{"action":"created","comment":{"url":"https://api.github.com/repos/apache/flink/pulls/comments/28412749","id":28412749,"diff_hunk":"@@ -0,0 +1,178 @@\n+/*\n+ * Licensed to the Apache Software Foundation (ASF) under one\n+ * or more contributor license agreements.  See the NOTICE file\n+ * distributed with this work for additional information\n+ * regarding copyright ownership.  The ASF licenses this file\n+ * to you under the Apache License, Version 2.0 (the\n+ * \"License\"); you may not use this file except in compliance\n+ * with the License.  You may obtain a copy of the License at\n+ *\n+ *     http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+\n+package org.apache.flink.ml.preprocessing\n+\n+import java.lang.Iterable\n+import breeze.linalg\n+import breeze.linalg.DenseVector\n+import breeze.numerics.sqrt\n+import breeze.numerics.sqrt._\n+import org.apache.flink.api.common.functions._\n+import org.apache.flink.api.scala._\n+import org.apache.flink.configuration.Configuration\n+import org.apache.flink.ml.common.{Parameter, ParameterMap, Transformer}\n+import org.apache.flink.ml.math.Breeze._\n+import org.apache.flink.ml.math.Vector\n+import org.apache.flink.ml.preprocessing.StandardScaler.{Mean, Std}\n+import org.apache.flink.util.Collector\n+\n+/** Scales observations, so that all features have mean equal to zero\n+  * and standard deviation equal to one\n+  *\n+  * This transformer takes a a Vector of values and maps it into the\n+  * scaled Vector that each feature has a user-specified mean and standard deviation.\n+  *\n+  * This transformer can be prepended to all [[Transformer]] and\n+  * [[org.apache.flink.ml.common.Learner]] implementations which expect an input of\n+  * [[Vector]].\n+  *\n+  * @example\n+  * {{{\n+  *                              val trainingDS: DataSet[Vector] = env.fromCollection(data)\n+  *\n+  *                              val transformer = StandardScaler().setMean(10.0).setStd(2.0)\n+  *\n+  *                              transformer.transform(trainingDS)\n+  * }}}\n+  *\n+  * =Parameters=\n+  *\n+  * - [[StandardScaler.Mean]]: The mean value of transformed data set; by default equal to 0\n+  * - [[StandardScaler.Std]]: The standard deviation of the transformed data set; by default\n+  * equal to 1\n+  */\n+class StandardScaler extends Transformer[Vector, Vector] with Serializable {\n+\n+  def setMean(mu: Double): StandardScaler = {\n+    parameters.add(Mean, mu)\n+    this\n+  }\n+\n+  def setStd(std: Double): StandardScaler = {\n+    parameters.add(Std, std)\n+    this\n+  }\n+\n+  override def transform(input: DataSet[Vector], parameters: ParameterMap):\n+  DataSet[Vector] = {\n+    val resultingParameters = this.parameters ++ parameters\n+    val mean = resultingParameters(Mean)\n+    val std = resultingParameters(Std)\n+\n+    val featureMetrics = extractFeatureMetrics(input)\n+\n+    input.map(new RichMapFunction[Vector, Vector]() {\n+\n+      var broadcastMeanSet: Vector = null\n+      var broadcastStdSet: Vector = null\n+\n+      override def open(parameters: Configuration): Unit = {\n+        val broadcastedMetrics = getRuntimeContext().getBroadcastVariable[(Vector,\n+          Vector)](\"broadcastedMetrics\").get(0)\n+        broadcastMeanSet = broadcastedMetrics._1\n+        broadcastStdSet = broadcastedMetrics._2\n+      }\n+\n+      override def map(vector: Vector): Vector = {\n+        var myVector = vector.asBreeze\n+\n+        myVector :-= broadcastMeanSet.asBreeze\n+        myVector :/= broadcastStdSet.asBreeze\n+        myVector = (myVector :* std) :+ mean\n+        return myVector.fromBreeze\n+      }\n+    }).withBroadcastSet(featureMetrics, \"broadcastedMetrics\")\n+  }\n+\n+  /** Calculates in one pass over the data the features' mean and standard deviation.\n+    * For the calculation of the Standard deviation with one pas over the data,\n+    * the Youngs & Cramer algorithm was used\n+    *\n+    * @param dataSet The data set for which we want to calculate mean and variance\n+    * @return DataSet of Tuple2<featuresMeanVector,featuresStdVector>\n+    */\n+  private def extractFeatureMetrics(dataSet: DataSet[Vector]) = {","path":"flink-staging/flink-ml/src/main/scala/org/apache/flink/ml/preprocessing/StandardScaler.scala","position":110,"original_position":110,"commit_id":"9a83e2d690c7852c241fd9d9c93bd5c39f6159d9","original_commit_id":"9a83e2d690c7852c241fd9d9c93bd5c39f6159d9","user":{"login":"tillrohrmann","id":5756858,"avatar_url":"https://avatars.githubusercontent.com/u/5756858?v=3","gravatar_id":"","url":"https://api.github.com/users/tillrohrmann","html_url":"https://github.com/tillrohrmann","followers_url":"https://api.github.com/users/tillrohrmann/followers","following_url":"https://api.github.com/users/tillrohrmann/following{/other_user}","gists_url":"https://api.github.com/users/tillrohrmann/gists{/gist_id}","starred_url":"https://api.github.com/users/tillrohrmann/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tillrohrmann/subscriptions","organizations_url":"https://api.github.com/users/tillrohrmann/orgs","repos_url":"https://api.github.com/users/tillrohrmann/repos","events_url":"https://api.github.com/users/tillrohrmann/events{/privacy}","received_events_url":"https://api.github.com/users/tillrohrmann/received_events","type":"User","site_admin":false},"body":"By adding explicitly the return type of Scala methods, here ```DataSet[(Vector, Vector)]```, you can check that the method is actually returning this type.","created_at":"2015-04-15T11:51:11Z","updated_at":"2015-04-15T11:51:11Z","html_url":"https://github.com/apache/flink/pull/579#discussion_r28412749","pull_request_url":"https://api.github.com/repos/apache/flink/pulls/579","_links":{"self":{"href":"https://api.github.com/repos/apache/flink/pulls/comments/28412749"},"html":{"href":"https://github.com/apache/flink/pull/579#discussion_r28412749"},"pull_request":{"href":"https://api.github.com/repos/apache/flink/pulls/579"}}},"pull_request":{"url":"https://api.github.com/repos/apache/flink/pulls/579","id":32848259,"html_url":"https://github.com/apache/flink/pull/579","diff_url":"https://github.com/apache/flink/pull/579.diff","patch_url":"https://github.com/apache/flink/pull/579.patch","issue_url":"https://api.github.com/repos/apache/flink/issues/579","number":579,"state":"open","locked":false,"title":"Ml branch","user":{"login":"fobeligi","id":6593112,"avatar_url":"https://avatars.githubusercontent.com/u/6593112?v=3","gravatar_id":"","url":"https://api.github.com/users/fobeligi","html_url":"https://github.com/fobeligi","followers_url":"https://api.github.com/users/fobeligi/followers","following_url":"https://api.github.com/users/fobeligi/following{/other_user}","gists_url":"https://api.github.com/users/fobeligi/gists{/gist_id}","starred_url":"https://api.github.com/users/fobeligi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fobeligi/subscriptions","organizations_url":"https://api.github.com/users/fobeligi/orgs","repos_url":"https://api.github.com/users/fobeligi/repos","events_url":"https://api.github.com/users/fobeligi/events{/privacy}","received_events_url":"https://api.github.com/users/fobeligi/received_events","type":"User","site_admin":false},"body":"[FLINK-1809] Add standard scaler to ML library\r\nImplementation of StandardScaler and respective tests for FLINK-1809 JIRA.\r\n","created_at":"2015-04-08T10:48:38Z","updated_at":"2015-04-15T11:51:11Z","closed_at":null,"merged_at":null,"merge_commit_sha":"47313af999e2b78239a30528ed7f64249cbfa3f3","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/apache/flink/pulls/579/commits","review_comments_url":"https://api.github.com/repos/apache/flink/pulls/579/comments","review_comment_url":"https://api.github.com/repos/apache/flink/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/flink/issues/579/comments","statuses_url":"https://api.github.com/repos/apache/flink/statuses/9a83e2d690c7852c241fd9d9c93bd5c39f6159d9","head":{"label":"fobeligi:ml-branch","ref":"ml-branch","sha":"9a83e2d690c7852c241fd9d9c93bd5c39f6159d9","user":{"login":"fobeligi","id":6593112,"avatar_url":"https://avatars.githubusercontent.com/u/6593112?v=3","gravatar_id":"","url":"https://api.github.com/users/fobeligi","html_url":"https://github.com/fobeligi","followers_url":"https://api.github.com/users/fobeligi/followers","following_url":"https://api.github.com/users/fobeligi/following{/other_user}","gists_url":"https://api.github.com/users/fobeligi/gists{/gist_id}","starred_url":"https://api.github.com/users/fobeligi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fobeligi/subscriptions","organizations_url":"https://api.github.com/users/fobeligi/orgs","repos_url":"https://api.github.com/users/fobeligi/repos","events_url":"https://api.github.com/users/fobeligi/events{/privacy}","received_events_url":"https://api.github.com/users/fobeligi/received_events","type":"User","site_admin":false},"repo":{"id":28087574,"name":"incubator-flink","full_name":"fobeligi/incubator-flink","owner":{"login":"fobeligi","id":6593112,"avatar_url":"https://avatars.githubusercontent.com/u/6593112?v=3","gravatar_id":"","url":"https://api.github.com/users/fobeligi","html_url":"https://github.com/fobeligi","followers_url":"https://api.github.com/users/fobeligi/followers","following_url":"https://api.github.com/users/fobeligi/following{/other_user}","gists_url":"https://api.github.com/users/fobeligi/gists{/gist_id}","starred_url":"https://api.github.com/users/fobeligi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fobeligi/subscriptions","organizations_url":"https://api.github.com/users/fobeligi/orgs","repos_url":"https://api.github.com/users/fobeligi/repos","events_url":"https://api.github.com/users/fobeligi/events{/privacy}","received_events_url":"https://api.github.com/users/fobeligi/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/fobeligi/incubator-flink","description":"Mirror of Apache Flink","fork":true,"url":"https://api.github.com/repos/fobeligi/incubator-flink","forks_url":"https://api.github.com/repos/fobeligi/incubator-flink/forks","keys_url":"https://api.github.com/repos/fobeligi/incubator-flink/keys{/key_id}","collaborators_url":"https://api.github.com/repos/fobeligi/incubator-flink/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/fobeligi/incubator-flink/teams","hooks_url":"https://api.github.com/repos/fobeligi/incubator-flink/hooks","issue_events_url":"https://api.github.com/repos/fobeligi/incubator-flink/issues/events{/number}","events_url":"https://api.github.com/repos/fobeligi/incubator-flink/events","assignees_url":"https://api.github.com/repos/fobeligi/incubator-flink/assignees{/user}","branches_url":"https://api.github.com/repos/fobeligi/incubator-flink/branches{/branch}","tags_url":"https://api.github.com/repos/fobeligi/incubator-flink/tags","blobs_url":"https://api.github.com/repos/fobeligi/incubator-flink/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/fobeligi/incubator-flink/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/fobeligi/incubator-flink/git/refs{/sha}","trees_url":"https://api.github.com/repos/fobeligi/incubator-flink/git/trees{/sha}","statuses_url":"https://api.github.com/repos/fobeligi/incubator-flink/statuses/{sha}","languages_url":"https://api.github.com/repos/fobeligi/incubator-flink/languages","stargazers_url":"https://api.github.com/repos/fobeligi/incubator-flink/stargazers","contributors_url":"https://api.github.com/repos/fobeligi/incubator-flink/contributors","subscribers_url":"https://api.github.com/repos/fobeligi/incubator-flink/subscribers","subscription_url":"https://api.github.com/repos/fobeligi/incubator-flink/subscription","commits_url":"https://api.github.com/repos/fobeligi/incubator-flink/commits{/sha}","git_commits_url":"https://api.github.com/repos/fobeligi/incubator-flink/git/commits{/sha}","comments_url":"https://api.github.com/repos/fobeligi/incubator-flink/comments{/number}","issue_comment_url":"https://api.github.com/repos/fobeligi/incubator-flink/issues/comments{/number}","contents_url":"https://api.github.com/repos/fobeligi/incubator-flink/contents/{+path}","compare_url":"https://api.github.com/repos/fobeligi/incubator-flink/compare/{base}...{head}","merges_url":"https://api.github.com/repos/fobeligi/incubator-flink/merges","archive_url":"https://api.github.com/repos/fobeligi/incubator-flink/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/fobeligi/incubator-flink/downloads","issues_url":"https://api.github.com/repos/fobeligi/incubator-flink/issues{/number}","pulls_url":"https://api.github.com/repos/fobeligi/incubator-flink/pulls{/number}","milestones_url":"https://api.github.com/repos/fobeligi/incubator-flink/milestones{/number}","notifications_url":"https://api.github.com/repos/fobeligi/incubator-flink/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/fobeligi/incubator-flink/labels{/name}","releases_url":"https://api.github.com/repos/fobeligi/incubator-flink/releases{/id}","created_at":"2014-12-16T13:11:52Z","updated_at":"2015-01-08T10:07:51Z","pushed_at":"2015-04-14T16:51:28Z","git_url":"git://github.com/fobeligi/incubator-flink.git","ssh_url":"git@github.com:fobeligi/incubator-flink.git","clone_url":"https://github.com/fobeligi/incubator-flink.git","svn_url":"https://github.com/fobeligi/incubator-flink","homepage":null,"size":45808,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"69a400fadd258fe0a1ff0b5670a3611fda4c1cdf","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":20587599,"name":"flink","full_name":"apache/flink","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/flink","description":"Mirror of Apache Flink","fork":false,"url":"https://api.github.com/repos/apache/flink","forks_url":"https://api.github.com/repos/apache/flink/forks","keys_url":"https://api.github.com/repos/apache/flink/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/flink/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/flink/teams","hooks_url":"https://api.github.com/repos/apache/flink/hooks","issue_events_url":"https://api.github.com/repos/apache/flink/issues/events{/number}","events_url":"https://api.github.com/repos/apache/flink/events","assignees_url":"https://api.github.com/repos/apache/flink/assignees{/user}","branches_url":"https://api.github.com/repos/apache/flink/branches{/branch}","tags_url":"https://api.github.com/repos/apache/flink/tags","blobs_url":"https://api.github.com/repos/apache/flink/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/flink/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/flink/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/flink/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/flink/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/flink/languages","stargazers_url":"https://api.github.com/repos/apache/flink/stargazers","contributors_url":"https://api.github.com/repos/apache/flink/contributors","subscribers_url":"https://api.github.com/repos/apache/flink/subscribers","subscription_url":"https://api.github.com/repos/apache/flink/subscription","commits_url":"https://api.github.com/repos/apache/flink/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/flink/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/flink/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/flink/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/flink/contents/{+path}","compare_url":"https://api.github.com/repos/apache/flink/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/flink/merges","archive_url":"https://api.github.com/repos/apache/flink/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/flink/downloads","issues_url":"https://api.github.com/repos/apache/flink/issues{/number}","pulls_url":"https://api.github.com/repos/apache/flink/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/flink/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/flink/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/flink/labels{/name}","releases_url":"https://api.github.com/repos/apache/flink/releases{/id}","created_at":"2014-06-07T07:00:10Z","updated_at":"2015-04-15T09:38:58Z","pushed_at":"2015-04-15T10:19:34Z","git_url":"git://github.com/apache/flink.git","ssh_url":"git@github.com:apache/flink.git","clone_url":"https://github.com/apache/flink.git","svn_url":"https://github.com/apache/flink","homepage":null,"size":3073793,"stargazers_count":360,"watchers_count":360,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":210,"mirror_url":"git://git.apache.org/incubator-flink.git","open_issues_count":45,"forks":210,"open_issues":45,"watchers":360,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/flink/pulls/579"},"html":{"href":"https://github.com/apache/flink/pull/579"},"issue":{"href":"https://api.github.com/repos/apache/flink/issues/579"},"comments":{"href":"https://api.github.com/repos/apache/flink/issues/579/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/flink/pulls/579/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/flink/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/flink/pulls/579/commits"},"statuses":{"href":"https://api.github.com/repos/apache/flink/statuses/9a83e2d690c7852c241fd9d9c93bd5c39f6159d9"}}}}