{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/29692515","id":29692515,"diff_hunk":"@@ -0,0 +1,847 @@\n+/*\n+ * Licensed to the Apache Software Foundation (ASF) under one or more\n+ * contributor license agreements.  See the NOTICE file distributed with\n+ * this work for additional information regarding copyright ownership.\n+ * The ASF licenses this file to You under the Apache License, Version 2.0\n+ * (the \"License\"); you may not use this file except in compliance with\n+ * the License.  You may obtain a copy of the License at\n+ *\n+ *    http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+\n+package org.apache.spark.sql.hive.execution\n+\n+import java.io.File\n+import java.util.{Locale, TimeZone}\n+\n+import org.apache.spark.sql.hive.test.TestHive\n+import org.apache.spark.sql.hive.test.TestHive._\n+import org.apache.spark.util.Utils\n+import org.scalatest.BeforeAndAfter\n+\n+/**\n+ * The test suite for window functions. To actually compare results with Hive,\n+ * every test should be created by `createQueryTest`. Because we are reusing tables\n+ * for different tests and there are a few properties needed to let Hive generate golden\n+ * files, every `createQueryTest` calls should explicitly set `reset` to `false`.\n+ */\n+abstract class HiveWindowFunctionQueryBaseSuite extends HiveComparisonTest with BeforeAndAfter {\n+  private val originalTimeZone = TimeZone.getDefault\n+  private val originalLocale = Locale.getDefault\n+  private val testTempDir = Utils.createTempDir()\n+  import org.apache.spark.sql.hive.test.TestHive.implicits._\n+\n+  override def beforeAll() {\n+    TestHive.cacheTables = true\n+    // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)\n+    TimeZone.setDefault(TimeZone.getTimeZone(\"America/Los_Angeles\"))\n+    // Add Locale setting\n+    Locale.setDefault(Locale.US)\n+\n+    // Create the table used in windowing.q\n+    sql(\"DROP TABLE IF EXISTS part\")\n+    sql(\n+      \"\"\"\n+        |CREATE TABLE part(\n+        |  p_partkey INT,\n+        |  p_name STRING,\n+        |  p_mfgr STRING,\n+        |  p_brand STRING,\n+        |  p_type STRING,\n+        |  p_size INT,\n+        |  p_container STRING,\n+        |  p_retailprice DOUBLE,\n+        |  p_comment STRING)\n+      \"\"\".stripMargin)\n+    val testData1 = TestHive.getHiveFile(\"data/files/part_tiny.txt\").getCanonicalPath\n+    sql(\n+      s\"\"\"\n+        |LOAD DATA LOCAL INPATH '$testData1' overwrite into table part\n+      \"\"\".stripMargin)\n+\n+    sql(\"DROP TABLE IF EXISTS over1k\")\n+    sql(\n+      \"\"\"\n+        |create table over1k(\n+        |  t tinyint,\n+        |  si smallint,\n+        |  i int,\n+        |  b bigint,\n+        |  f float,\n+        |  d double,\n+        |  bo boolean,\n+        |  s string,\n+        |  ts timestamp,\n+        |  dec decimal(4,2),\n+        |  bin binary)\n+        |row format delimited\n+        |fields terminated by '|'\n+      \"\"\".stripMargin)\n+    val testData2 = TestHive.getHiveFile(\"data/files/over1k\").getCanonicalPath\n+    sql(\n+      s\"\"\"\n+        |LOAD DATA LOCAL INPATH '$testData2' overwrite into table over1k\n+      \"\"\".stripMargin)\n+\n+    // The following settings are used for generating golden files with Hive.\n+    // We have to use kryo to correctly let Hive serialize plans with window functions.\n+    // This is used to generate golden files.\n+    sql(\"set hive.plan.serialization.format=kryo\")\n+    // Explicitly set fs to local fs.\n+    sql(s\"set fs.default.name=file://$testTempDir/\")\n+    // Ask Hive to run jobs in-process as a single map and reduce task.\n+    sql(\"set mapred.job.tracker=local\")\n+  }\n+\n+  override def afterAll() {\n+    TestHive.cacheTables = false\n+    TimeZone.setDefault(originalTimeZone)\n+    Locale.setDefault(originalLocale)\n+    TestHive.reset()\n+  }\n+\n+  /////////////////////////////////////////////////////////////////////////////\n+  // Tests based on windowing_multipartitioning.q\n+  // Results of the original query file are not deterministic.\n+  /////////////////////////////////////////////////////////////////////////////\n+  createQueryTest(\"windowing_multipartitioning.q (deterministic) 1\",\n+    s\"\"\"\n+      |select s,\n+      |rank() over (partition by s order by si) r,\n+      |sum(b) over (partition by s order by si) sum\n+      |from over1k\n+      |order by s, r, sum;\n+    \"\"\".stripMargin, reset = false)\n+\n+  /* timestamp comparison issue with Hive?\n+  createQueryTest(\"windowing_multipartitioning.q (deterministic) 2\",\n+    s\"\"\"\n+      |select s,\n+      |rank() over (partition by s order by dec desc) r,\n+      |sum(b) over (partition by s order by ts desc) as sum\n+      |from over1k\n+      |where s = 'tom allen' or s = 'bob steinbeck'\n+      |order by s, r, sum;\n+     \"\"\".stripMargin, reset = false)\n+  */\n+\n+  createQueryTest(\"windowing_multipartitioning.q (deterministic) 3\",\n+    s\"\"\"\n+      |select s, sum(i) over (partition by s), sum(f) over (partition by si)\n+      |from over1k where s = 'tom allen' or s = 'bob steinbeck';\n+     \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing_multipartitioning.q (deterministic) 4\",\n+    s\"\"\"\n+      |select s, rank() over (partition by s order by bo),\n+      |rank() over (partition by si order by bin desc) from over1k\n+      |where s = 'tom allen' or s = 'bob steinbeck';\n+     \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing_multipartitioning.q (deterministic) 5\",\n+    s\"\"\"\n+      |select s, sum(f) over (partition by i), row_number() over (order by f)\n+      |from over1k where s = 'tom allen' or s = 'bob steinbeck';\n+     \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing_multipartitioning.q (deterministic) 6\",\n+    s\"\"\"\n+      |select s, rank() over w1,\n+      |rank() over w2\n+      |from over1k\n+      |where s = 'tom allen' or s = 'bob steinbeck'\n+      |window\n+      |w1 as (partition by s order by dec),\n+      |w2 as (partition by si order by f) ;\n+     \"\"\".stripMargin, reset = false)\n+\n+  /////////////////////////////////////////////////////////////////////////////\n+  // Tests based on windowing_navfn.q\n+  // Results of the original query file are not deterministic.\n+  // Also, the original query of\n+  // select i, lead(s) over (partition by bin order by d,i desc) from over1k ;\n+  /////////////////////////////////////////////////////////////////////////////\n+  createQueryTest(\"windowing_navfn.q (deterministic)\",\n+    s\"\"\"\n+      |select s, row_number() over (partition by d order by dec) rn from over1k\n+      |order by s, rn desc;\n+      |select i, lead(s) over (partition by cast(bin as string) order by d,i desc) as l\n+      |from over1k\n+      |order by i desc, l;\n+      |select i, lag(dec) over (partition by i order by s,i,dec) l from over1k\n+      |order by i, l;\n+      |select s, last_value(t) over (partition by d order by f) l from over1k\n+      |order by s, l;\n+      |select s, first_value(s) over (partition by bo order by s) f from over1k\n+      |order by s, f;\n+      |select t, s, i, last_value(i) over (partition by t order by s)\n+      |from over1k where (s = 'oscar allen' or s = 'oscar carson') and t = 10;\n+     \"\"\".stripMargin, reset = false)\n+\n+  /////////////////////////////////////////////////////////////////////////////\n+  // Tests based on windowing_ntile.q\n+  // Results of the original query file are not deterministic.\n+  /////////////////////////////////////////////////////////////////////////////\n+  createQueryTest(\"windowing_ntile.q (deterministic)\",\n+    s\"\"\"\n+      |select i, ntile(10) over (partition by s order by i) n from over1k\n+      |order by i, n;\n+      |select s, ntile(100) over (partition by i order by s) n from over1k\n+      |order by s, n;\n+      |select f, ntile(4) over (partition by d order by f) n from over1k\n+      |order by f, n;\n+      |select d, ntile(1000) over (partition by dec order by d) n from over1k\n+      |order by d, n;\n+     \"\"\".stripMargin, reset = false)\n+\n+  /////////////////////////////////////////////////////////////////////////////\n+  // Tests based on windowing_udaf.q\n+  // Results of the original query file are not deterministic.\n+  /////////////////////////////////////////////////////////////////////////////\n+  createQueryTest(\"windowing_udaf.q (deterministic)\",\n+    s\"\"\"\n+      |select s, min(i) over (partition by s) m from over1k\n+      |order by s, m;\n+      |select s, avg(f) over (partition by si order by s) a from over1k\n+      |order by s, a;\n+      |select s, avg(i) over (partition by t, b order by s) a from over1k\n+      |order by s, a;\n+      |select max(i) over w m from over1k\n+      |order by m window w as (partition by f) ;\n+      |select s, avg(d) over (partition by t order by f) a from over1k\n+      |order by s, a;\n+     \"\"\".stripMargin, reset = false)\n+\n+  /////////////////////////////////////////////////////////////////////////////\n+  // Tests based on windowing_windowspec.q\n+  // Results of the original query file are not deterministic.\n+  /////////////////////////////////////////////////////////////////////////////\n+  createQueryTest(\"windowing_windowspec.q (deterministic)\",\n+    s\"\"\"\n+      |select s, sum(b) over (partition by i order by s,b rows unbounded preceding) as sum\n+      |from over1k order by s, sum;\n+      |select s, sum(f) over (partition by d order by s,f rows unbounded preceding) as sum\n+      |from over1k order by s, sum;\n+      |select s, sum(f) over\n+      |(partition by ts order by f range between current row and unbounded following) as sum\n+      |from over1k order by s, sum;\n+      |select s, avg(f)\n+      |over (partition by ts order by s,f rows between current row and 5 following) avg\n+      |from over1k order by s, avg;\n+      |select s, avg(d) over\n+      |(partition by t order by s,d desc rows between 5 preceding and 5 following) avg\n+      |from over1k order by s, avg;\n+      |select s, sum(i) over(partition by ts order by s) sum from over1k\n+      |order by s, sum;\n+      |select f, sum(f) over\n+      |(partition by ts order by f range between unbounded preceding and current row) sum\n+      |from over1k order by f, sum;\n+      |select s, i, round(avg(d) over (partition by s order by i) / 10.0 , 2) avg\n+      |from over1k order by s, i, avg;\n+      |select s, i, round((avg(d) over  w1 + 10.0) - (avg(d) over w1 - 10.0),2) avg\n+      |from over1k\n+      |order by s, i, avg window w1 as (partition by s order by i);\n+     \"\"\".stripMargin, reset = false)\n+\n+  /////////////////////////////////////////////////////////////////////////////\n+  // Tests based on windowing_rank.q\n+  // Results of the original query file are not deterministic.\n+  /////////////////////////////////////////////////////////////////////////////\n+  createQueryTest(\"windowing_rank.q (deterministic) 1\",\n+    s\"\"\"\n+      |select s, rank() over (partition by f order by t) r from over1k order by s, r;\n+      |select s, dense_rank() over (partition by ts order by i,s desc) as r from over1k\n+      |order by s desc, r desc;\n+      |select s, cume_dist() over (partition by bo order by b,s) cd from over1k\n+      |order by s, cd;\n+      |select s, percent_rank() over (partition by dec order by f) r from over1k\n+      |order by s desc, r desc;\n+     \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing_rank.q (deterministic) 2\",\n+    s\"\"\"\n+      |select ts, dec, rnk\n+      |from\n+      |  (select ts, dec,\n+      |          rank() over (partition by ts order by dec)  as rnk\n+      |          from\n+      |            (select other.ts, other.dec\n+      |             from over1k other\n+      |             join over1k on (other.b = over1k.b)\n+      |            ) joined\n+      |  ) ranked\n+      |where rnk =  1\n+      |order by ts, dec, rnk;\n+     \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing_rank.q (deterministic) 3\",\n+    s\"\"\"\n+      |select ts, dec, rnk\n+      |from\n+      |  (select ts, dec,\n+      |          rank() over (partition by ts order by dec)  as rnk\n+      |          from\n+      |            (select other.ts, other.dec\n+      |             from over1k other\n+      |             join over1k on (other.b = over1k.b)\n+      |            ) joined\n+      |  ) ranked\n+      |where dec = 89.5\n+      |order by ts, dec, rnk;\n+     \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing_rank.q (deterministic) 4\",\n+    s\"\"\"\n+      |select ts, dec, rnk\n+      |from\n+      |  (select ts, dec,\n+      |          rank() over (partition by ts order by dec)  as rnk\n+      |          from\n+      |            (select other.ts, other.dec\n+      |             from over1k other\n+      |             join over1k on (other.b = over1k.b)\n+      |             where other.t < 10\n+      |            ) joined\n+      |  ) ranked\n+      |where rnk = 1\n+      |order by ts, dec, rnk;\n+     \"\"\".stripMargin, reset = false)\n+\n+  /////////////////////////////////////////////////////////////////////////////\n+  // Tests from windowing.q\n+  // We port tests in windowing.q to here because this query file contains too\n+  // many tests and the syntax of test \"-- 7. testJoinWithWindowingAndPTF\"\n+  // is not supported right now.\n+  /////////////////////////////////////////////////////////////////////////////\n+  createQueryTest(\"windowing.q -- 1. testWindowing\",\n+    s\"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |rank() over(distribute by p_mfgr sort by p_name) as r,\n+      |dense_rank() over(distribute by p_mfgr sort by p_name) as dr,\n+      |sum(p_retailprice) over\n+      |(distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) as s1\n+      |from part\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 2. testGroupByWithPartitioning\",\n+    s\"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |min(p_retailprice),\n+      |rank() over(distribute by p_mfgr sort by p_name)as r,\n+      |dense_rank() over(distribute by p_mfgr sort by p_name) as dr,\n+      |p_size, p_size - lag(p_size,1,p_size) over(distribute by p_mfgr sort by p_name) as deltaSz\n+      |from part\n+      |group by p_mfgr, p_name, p_size\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 3. testGroupByHavingWithSWQ\",\n+    s\"\"\"\n+      |select p_mfgr, p_name, p_size, min(p_retailprice),\n+      |rank() over(distribute by p_mfgr sort by p_name) as r,\n+      |dense_rank() over(distribute by p_mfgr sort by p_name) as dr,\n+      |p_size, p_size - lag(p_size,1,p_size) over(distribute by p_mfgr sort by p_name) as deltaSz\n+      |from part\n+      |group by p_mfgr, p_name, p_size\n+      |having p_size > 0\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 4. testCount\",\n+    s\"\"\"\n+      |select p_mfgr, p_name,\n+      |count(p_size) over(distribute by p_mfgr sort by p_name) as cd\n+      |from part\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 5. testCountWithWindowingUDAF\",\n+    s\"\"\"\n+      |select p_mfgr, p_name,\n+      |rank() over(distribute by p_mfgr sort by p_name) as r,\n+      |dense_rank() over(distribute by p_mfgr sort by p_name) as dr,\n+      |count(p_size) over(distribute by p_mfgr sort by p_name) as cd,\n+      |p_retailprice, sum(p_retailprice) over (distribute by p_mfgr sort by p_name\n+      |                                  rows between unbounded preceding and current row) as s1,\n+      |p_size, p_size - lag(p_size,1,p_size) over(distribute by p_mfgr sort by p_name) as deltaSz\n+      |from part\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 6. testCountInSubQ\",\n+    s\"\"\"\n+      |select sub1.r, sub1.dr, sub1.cd, sub1.s1, sub1.deltaSz\n+      |from (select p_mfgr, p_name,\n+      |rank() over(distribute by p_mfgr sort by p_name) as r,\n+      |dense_rank() over(distribute by p_mfgr sort by p_name) as dr,\n+      |count(p_size) over(distribute by p_mfgr sort by p_name) as cd,\n+      |p_retailprice, sum(p_retailprice) over (distribute by p_mfgr sort by p_name\n+      |                                  rows between unbounded preceding and current row) as s1,\n+      |p_size, p_size - lag(p_size,1,p_size) over(distribute by p_mfgr sort by p_name) as deltaSz\n+      |from part\n+      |) sub1\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 8. testMixedCaseAlias\",\n+    s\"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |rank() over(distribute by p_mfgr sort by p_name, p_size desc) as R\n+      |from part\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 9. testHavingWithWindowingNoGBY\",\n+    s\"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |rank() over(distribute by p_mfgr sort by p_name) as r,\n+      |dense_rank() over(distribute by p_mfgr sort by p_name) as dr,\n+      |sum(p_retailprice) over (distribute by p_mfgr sort by p_name\n+      |                        rows between unbounded preceding and current row)  as s1\n+      |from part\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 10. testHavingWithWindowingCondRankNoGBY\",\n+    s\"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |rank() over(distribute by p_mfgr sort by p_name) as r,\n+      |dense_rank() over(distribute by p_mfgr sort by p_name) as dr,\n+      |sum(p_retailprice) over (distribute by p_mfgr sort by p_name\n+      |                        rows between unbounded preceding and current row) as s1\n+      |from part\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 11. testFirstLast\",\n+    s\"\"\"\n+      |select  p_mfgr,p_name, p_size,\n+      |sum(p_size) over (distribute by p_mfgr sort by p_name\n+      |rows between current row and current row) as s2,\n+      |first_value(p_size) over w1  as f,\n+      |last_value(p_size, false) over w1  as l\n+      |from part\n+      |window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following)\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 12. testFirstLastWithWhere\",\n+    s\"\"\"\n+      |select  p_mfgr,p_name, p_size,\n+      |rank() over(distribute by p_mfgr sort by p_name) as r,\n+      |sum(p_size) over (distribute by p_mfgr sort by p_name\n+      |rows between current row and current row) as s2,\n+      |first_value(p_size) over w1 as f,\n+      |last_value(p_size, false) over w1 as l\n+      |from part\n+      |where p_mfgr = 'Manufacturer#3'\n+      |window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following)\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 13. testSumWindow\",\n+    s\"\"\"\n+      |select  p_mfgr,p_name, p_size,\n+      |sum(p_size) over w1 as s1,\n+      |sum(p_size) over (distribute by p_mfgr  sort by p_name\n+      |rows between current row and current row)  as s2\n+      |from part\n+      |window w1 as (distribute by p_mfgr  sort by p_name rows between 2 preceding and 2 following)\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 14. testNoSortClause\",\n+    s\"\"\"\n+      |select  p_mfgr,p_name, p_size,\n+      |rank() over(distribute by p_mfgr sort by p_name) as r,\n+      |dense_rank() over(distribute by p_mfgr sort by p_name) as dr\n+      |from part\n+      |window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following)\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 15. testExpressions\",\n+    s\"\"\"\n+      |select  p_mfgr,p_name, p_size,\n+      |rank() over(distribute by p_mfgr sort by p_name) as r,\n+      |dense_rank() over(distribute by p_mfgr sort by p_name) as dr,\n+      |cume_dist() over(distribute by p_mfgr sort by p_name) as cud,\n+      |percent_rank() over(distribute by p_mfgr sort by p_name) as pr,\n+      |ntile(3) over(distribute by p_mfgr sort by p_name) as nt,\n+      |count(p_size) over(distribute by p_mfgr sort by p_name) as ca,\n+      |avg(p_size) over(distribute by p_mfgr sort by p_name) as avg,\n+      |stddev(p_size) over(distribute by p_mfgr sort by p_name) as st,\n+      |first_value(p_size % 5) over(distribute by p_mfgr sort by p_name) as fv,\n+      |last_value(p_size) over(distribute by p_mfgr sort by p_name) as lv,\n+      |first_value(p_size) over w1  as fvW1\n+      |from part\n+      |window w1 as (distribute by p_mfgr sort by p_mfgr, p_name\n+      |             rows between 2 preceding and 2 following)\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 16. testMultipleWindows\",\n+    s\"\"\"\n+      |select  p_mfgr,p_name, p_size,\n+      |rank() over(distribute by p_mfgr sort by p_name) as r,\n+      |dense_rank() over(distribute by p_mfgr sort by p_name) as dr,\n+      |cume_dist() over(distribute by p_mfgr sort by p_name) as cud,\n+      |sum(p_size) over (distribute by p_mfgr sort by p_name\n+      |range between unbounded preceding and current row) as s1,\n+      |sum(p_size) over (distribute by p_mfgr sort by p_size\n+      |range between 5 preceding and current row) as s2,\n+      |first_value(p_size) over w1  as fv1\n+      |from part\n+      |window w1 as (distribute by p_mfgr sort by p_mfgr, p_name\n+      |             rows between 2 preceding and 2 following)\n+    \"\"\".stripMargin, reset = false)\n+\n+\n+  createQueryTest(\"windowing.q -- 17. testCountStar\",\n+    s\"\"\"\n+      |select  p_mfgr,p_name, p_size,\n+      |count(*) over(distribute by p_mfgr sort by p_name ) as c,\n+      |count(p_size) over(distribute by p_mfgr sort by p_name) as ca,\n+      |first_value(p_size) over w1  as fvW1\n+      |from part\n+      |window w1 as (distribute by p_mfgr sort by p_mfgr, p_name\n+      |             rows between 2 preceding and 2 following)\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 18. testUDAFs\",\n+    s\"\"\"\n+      |select  p_mfgr,p_name, p_size,\n+      |sum(p_retailprice) over w1 as s,\n+      |min(p_retailprice) over w1 as mi,\n+      |max(p_retailprice) over w1 as ma,\n+      |avg(p_retailprice) over w1 as ag\n+      |from part\n+      |window w1 as (distribute by p_mfgr sort by p_mfgr, p_name\n+      |             rows between 2 preceding and 2 following)\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 19. testUDAFsWithGBY\",\n+    \"\"\"\n+      |select  p_mfgr,p_name, p_size, p_retailprice,\n+      |sum(p_retailprice) over w1 as s,\n+      |min(p_retailprice) as mi ,\n+      |max(p_retailprice) as ma ,\n+      |avg(p_retailprice) over w1 as ag\n+      |from part\n+      |group by p_mfgr,p_name, p_size, p_retailprice\n+      |window w1 as (distribute by p_mfgr sort by p_mfgr, p_name\n+      |             rows between 2 preceding and 2 following);\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 20. testSTATs\",\n+    \"\"\"\n+      |select  p_mfgr,p_name, p_size,\n+      |stddev(p_retailprice) over w1 as sdev,\n+      |stddev_pop(p_retailprice) over w1 as sdev_pop,\n+      |collect_set(p_size) over w1 as uniq_size,\n+      |variance(p_retailprice) over w1 as var,\n+      |corr(p_size, p_retailprice) over w1 as cor,\n+      |covar_pop(p_size, p_retailprice) over w1 as covarp\n+      |from part\n+      |window w1 as (distribute by p_mfgr sort by p_mfgr, p_name\n+      |             rows between 2 preceding and 2 following)\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 21. testDISTs\",\n+    \"\"\"\n+      |select  p_mfgr,p_name, p_size,\n+      |histogram_numeric(p_retailprice, 5) over w1 as hist,\n+      |percentile(p_partkey, 0.5) over w1 as per,\n+      |row_number() over(distribute by p_mfgr sort by p_name) as rn\n+      |from part\n+      |window w1 as (distribute by p_mfgr sort by p_mfgr, p_name\n+      |             rows between 2 preceding and 2 following)\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 24. testLateralViews\",\n+    \"\"\"\n+      |select p_mfgr, p_name,\n+      |lv_col, p_size, sum(p_size) over w1   as s\n+      |from (select p_mfgr, p_name, p_size, array(1,2,3) arr from part) p\n+      |lateral view explode(arr) part_lv as lv_col\n+      |window w1 as (distribute by p_mfgr sort by p_size, lv_col\n+      |             rows between 2 preceding and current row)\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 26. testGroupByHavingWithSWQAndAlias\",\n+    \"\"\"\n+      |select p_mfgr, p_name, p_size, min(p_retailprice) as mi,\n+      |rank() over(distribute by p_mfgr sort by p_name) as r,\n+      |dense_rank() over(distribute by p_mfgr sort by p_name) as dr,\n+      |p_size, p_size - lag(p_size,1,p_size) over(distribute by p_mfgr sort by p_name) as deltaSz\n+      |from part\n+      |group by p_mfgr, p_name, p_size\n+      |having p_size > 0\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 27. testMultipleRangeWindows\",\n+    \"\"\"\n+      |select  p_mfgr,p_name, p_size,\n+      |sum(p_size) over (distribute by p_mfgr sort by p_size\n+      |range between 10 preceding and current row) as s2,\n+      |sum(p_size) over (distribute by p_mfgr sort by p_size\n+      |range between current row and 10 following )  as s1\n+      |from part\n+      |window w1 as (rows between 2 preceding and 2 following)\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 28. testPartOrderInUDAFInvoke\",\n+    \"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |sum(p_size) over (partition by p_mfgr  order by p_name\n+      |rows between 2 preceding and 2 following) as s\n+      |from part\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 29. testPartOrderInWdwDef\",\n+    \"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |sum(p_size) over w1 as s\n+      |from part\n+      |window w1 as (partition by p_mfgr  order by p_name\n+      |             rows between 2 preceding and 2 following)\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 30. testDefaultPartitioningSpecRules\",\n+    \"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |sum(p_size) over w1 as s,\n+      |sum(p_size) over w2 as s2\n+      |from part\n+      |window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following),\n+      |       w2 as (partition by p_mfgr order by p_name)\n+    \"\"\".stripMargin, reset = false)\n+\n+  /* p_name is not a numeric column. What is Hive's semantic?\n+  createQueryTest(\"windowing.q -- 31. testWindowCrossReference\",\n+    \"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |sum(p_size) over w1 as s1,\n+      |sum(p_size) over w2 as s2\n+      |from part\n+      |window w1 as (partition by p_mfgr order by p_name\n+      |             range between 2 preceding and 2 following),\n+      |       w2 as w1\n+    \"\"\".stripMargin, reset = false)\n+  */\n+  /*\n+  createQueryTest(\"windowing.q -- 32. testWindowInheritance\",\n+    \"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |sum(p_size) over w1 as s1,\n+      |sum(p_size) over w2 as s2\n+      |from part\n+      |window w1 as (partition by p_mfgr order by p_name\n+      |             range between 2 preceding and 2 following),\n+      |       w2 as (w1 rows between unbounded preceding and current row)\n+    \"\"\".stripMargin, reset = false)\n+  */\n+\n+  /* p_name is not a numeric column. What is Hive's semantic?\n+  createQueryTest(\"windowing.q -- 33. testWindowForwardReference\",\n+    \"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |sum(p_size) over w1 as s1,\n+      |sum(p_size) over w2 as s2,\n+      |sum(p_size) over w3 as s3\n+      |from part\n+      |window w1 as (distribute by p_mfgr sort by p_name\n+      |             range between 2 preceding and 2 following),\n+      |       w2 as w3,\n+      |       w3 as (distribute by p_mfgr sort by p_name\n+      |             range between unbounded preceding and current row)\n+    \"\"\".stripMargin, reset = false)\n+  */\n+  /*\n+  createQueryTest(\"windowing.q -- 34. testWindowDefinitionPropagation\",\n+    \"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |sum(p_size) over w1 as s1,\n+      |sum(p_size) over w2 as s2,\n+      |sum(p_size) over (w3 rows between 2 preceding and 2 following)  as s3\n+      |from part\n+      |window w1 as (distribute by p_mfgr sort by p_name\n+      |             range between 2 preceding and 2 following),\n+      |       w2 as w3,\n+      |       w3 as (distribute by p_mfgr sort by p_name\n+      |             range between unbounded preceding and current row)\n+    \"\"\".stripMargin, reset = false)\n+  */\n+\n+  /* Seems Hive evaluate SELECT DISTINCT before window functions?\n+  createQueryTest(\"windowing.q -- 35. testDistinctWithWindowing\",\n+    \"\"\"\n+      |select DISTINCT p_mfgr, p_name, p_size,\n+      |sum(p_size) over w1 as s\n+      |from part\n+      |window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following)\n+    \"\"\".stripMargin, reset = false)\n+  */\n+\n+  createQueryTest(\"windowing.q -- 36. testRankWithPartitioning\",\n+    \"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |rank() over (partition by p_mfgr order by p_name )  as r\n+      |from part\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 37. testPartitioningVariousForms\",\n+    \"\"\"\n+      |select p_mfgr,\n+      |round(sum(p_retailprice) over (partition by p_mfgr order by p_mfgr),2) as s1,\n+      |min(p_retailprice) over (partition by p_mfgr) as s2,\n+      |max(p_retailprice) over (distribute by p_mfgr sort by p_mfgr) as s3,\n+      |round(avg(p_retailprice) over (distribute by p_mfgr),2) as s4,\n+      |count(p_retailprice) over (cluster by p_mfgr ) as s5\n+      |from part\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 38. testPartitioningVariousForms2\",\n+    \"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |sum(p_retailprice) over (partition by p_mfgr, p_name order by p_mfgr, p_name\n+      |rows between unbounded preceding and current row) as s1,\n+      |min(p_retailprice) over (distribute by p_mfgr, p_name sort by p_mfgr, p_name\n+      |rows between unbounded preceding and current row) as s2,\n+      |max(p_retailprice) over (partition by p_mfgr, p_name order by p_name) as s3\n+      |from part\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 39. testUDFOnOrderCols\",\n+    \"\"\"\n+      |select p_mfgr, p_type, substr(p_type, 2) as short_ptype,\n+      |rank() over (partition by p_mfgr order by substr(p_type, 2))  as r\n+      |from part\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 40. testNoBetweenForRows\",\n+    \"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |sum(p_retailprice) over (distribute by p_mfgr sort by p_name rows unbounded preceding) as s1\n+      |from part\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 41. testNoBetweenForRange\",\n+    \"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |sum(p_retailprice) over (distribute by p_mfgr sort by p_size range unbounded preceding) as s1\n+      |from part\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 42. testUnboundedFollowingForRows\",\n+    \"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |sum(p_retailprice) over (distribute by p_mfgr sort by p_name\n+      |rows between current row and unbounded following) as s1\n+      |from part\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 43. testUnboundedFollowingForRange\",\n+    \"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |sum(p_retailprice) over (distribute by p_mfgr sort by p_size\n+      |range between current row and unbounded following) as s1\n+      |from part\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 44. testOverNoPartitionSingleAggregate\",\n+    \"\"\"\n+      |select p_name, p_retailprice,\n+      |round(avg(p_retailprice) over(),2)\n+      |from part\n+      |order by p_name\n+    \"\"\".stripMargin, reset = false)\n+}\n+\n+class HiveWindowFunctionQueryWithoutCodeGenSuite extends HiveWindowFunctionQueryBaseSuite {\n+  var originalCodegenEnabled: Boolean = _\n+  override def beforeAll(): Unit = {\n+    super.beforeAll()\n+    originalCodegenEnabled = conf.codegenEnabled\n+    sql(\"set spark.sql.codegen=false\")\n+  }\n+\n+  override def afterAll(): Unit = {\n+    sql(s\"set spark.sql.codegen=$originalCodegenEnabled\")\n+    super.afterAll()\n+  }\n+}\n+\n+abstract class HiveWindowFunctionQueryFileBaseSuite\n+  extends HiveCompatibilitySuite with BeforeAndAfter {\n+  private val originalTimeZone = TimeZone.getDefault\n+  private val originalLocale = Locale.getDefault\n+  private val testTempDir = Utils.createTempDir()\n+\n+  import org.apache.spark.sql.hive.test.TestHive.implicits._\n+\n+  override def beforeAll() {\n+    TestHive.cacheTables = true\n+    // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)\n+    TimeZone.setDefault(TimeZone.getTimeZone(\"America/Los_Angeles\"))\n+    // Add Locale setting\n+    Locale.setDefault(Locale.US)\n+\n+    // The following settings are used for generating golden files with Hive.\n+    // We have to use kryo to correctly let Hive serialize plans with window functions.\n+    // This is used to generate golden files.\n+    sql(\"set hive.plan.serialization.format=kryo\")","path":"sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveWindowFunctionQuerySuite.scala","position":786,"original_position":786,"commit_id":"206d9e9582b8b2b34ca0d77fa3a1101a07b25cb7","original_commit_id":"206d9e9582b8b2b34ca0d77fa3a1101a07b25cb7","user":{"login":"yhuai","id":2072857,"avatar_url":"https://avatars.githubusercontent.com/u/2072857?v=3","gravatar_id":"","url":"https://api.github.com/users/yhuai","html_url":"https://github.com/yhuai","followers_url":"https://api.github.com/users/yhuai/followers","following_url":"https://api.github.com/users/yhuai/following{/other_user}","gists_url":"https://api.github.com/users/yhuai/gists{/gist_id}","starred_url":"https://api.github.com/users/yhuai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/yhuai/subscriptions","organizations_url":"https://api.github.com/users/yhuai/orgs","repos_url":"https://api.github.com/users/yhuai/repos","events_url":"https://api.github.com/users/yhuai/events{/privacy}","received_events_url":"https://api.github.com/users/yhuai/received_events","type":"User","site_admin":false},"body":"I think if we use `javaXML`, developers do not need to download Hive jars published for Spark (those `0.13.1a` jars) and they can still use regular Hive distribution. \r\nBasically, if `javaXML` works, it does not require any extra setup. If we set it to kryo, developers have to follow the setup instructions in https://github.com/apache/spark/tree/master/sql.","created_at":"2015-05-05T17:35:13Z","updated_at":"2015-05-05T17:35:13Z","html_url":"https://github.com/apache/spark/pull/5604#discussion_r29692515","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/5604","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/29692515"},"html":{"href":"https://github.com/apache/spark/pull/5604#discussion_r29692515"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/5604"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/5604","id":33731513,"html_url":"https://github.com/apache/spark/pull/5604","diff_url":"https://github.com/apache/spark/pull/5604.diff","patch_url":"https://github.com/apache/spark/pull/5604.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/5604","number":5604,"state":"open","locked":false,"title":"[SPARK-1442][SQL] Window Function Support for Spark SQL","user":{"login":"guowei2","id":6811813,"avatar_url":"https://avatars.githubusercontent.com/u/6811813?v=3","gravatar_id":"","url":"https://api.github.com/users/guowei2","html_url":"https://github.com/guowei2","followers_url":"https://api.github.com/users/guowei2/followers","following_url":"https://api.github.com/users/guowei2/following{/other_user}","gists_url":"https://api.github.com/users/guowei2/gists{/gist_id}","starred_url":"https://api.github.com/users/guowei2/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/guowei2/subscriptions","organizations_url":"https://api.github.com/users/guowei2/orgs","repos_url":"https://api.github.com/users/guowei2/repos","events_url":"https://api.github.com/users/guowei2/events{/privacy}","received_events_url":"https://api.github.com/users/guowei2/received_events","type":"User","site_admin":false},"body":"Adding more information about the implementation...\r\n\r\nThis PR is adding the support of window functions to Spark SQL (specifically OVER and WINDOW clause). For every expression having a OVER clause, we use a WindowExpression as the container of a WindowFunction and the corresponding WindowSpecDefinition (the definition of a window frame, i.e. partition specification, order specification, and frame specification appearing in a OVER clause). \r\n# Implementation #\r\nThe high level work flow of the implementation is described as follows.\r\n\r\n*\tQuery parsing: In the query parse process, all WindowExpressions are originally placed in the projectList of a Project operator or the aggregateExpressions of an Aggregate operator. It makes our changes to simple and keep all of parsing rules for window functions at a single place (nodesToWindowSpecification). For the WINDOWclause in a query, we use a WithWindowDefinition as the container as the mapping from the name of a window specification to a WindowSpecDefinition. This changes is similar with our common table expression support.\r\n\r\n*\tAnalysis: The query analysis process has three steps for window functions.\r\n\r\n *\tResolve all WindowSpecReferences by replacing them with WindowSpecReferences according to the mapping table stored in the node of WithWindowDefinition. \r\n *\tResolve WindowFunctions in the projectList of a Project operator or the aggregateExpressions of an Aggregate operator. For this PR, we use Hive's functions for window functions because we will have a major refactoring of our internal UDAFs and it is better to switch our UDAFs after that refactoring work. \r\n *\tOnce we have resolved all WindowFunctions, we will use ResolveWindowFunction to extract WindowExpressions from projectList and aggregateExpressions and then create a Window operator for every distinct WindowSpecDefinition. With this choice, at the execution time, we can rely on the Exchange operator to do all of work on reorganizing the table and we do not need to worry about it in the physical Window operator. An example analyzed plan is shown as follows\r\n\r\n```\r\nsql(\"\"\"\r\nSELECT\r\n  year, country, product, sales,\r\n  avg(sales) over(partition by product) avg_product,\r\n  sum(sales) over(partition by country) sum_country\r\nFROM sales\r\nORDER BY year, country, product\r\n\"\"\").explain(true)\r\n\r\n== Analyzed Logical Plan ==\r\nSort [year#34 ASC,country#35 ASC,product#36 ASC], true\r\n Project [year#34,country#35,product#36,sales#37,avg_product#27,sum_country#28]\r\n  Window [year#34,country#35,product#36,sales#37,avg_product#27], [HiveWindowFunction#org.apache.hadoop.hive.ql.udf.generic.GenericUDAFSum(sales#37) WindowSpecDefinition [country#35], [], ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS sum_country#28], WindowSpecDefinition [country#35], [], ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING\r\n   Window [year#34,country#35,product#36,sales#37], [HiveWindowFunction#org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage(sales#37) WindowSpecDefinition [product#36], [], ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS avg_product#27], WindowSpecDefinition [product#36], [], ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING\r\n    Project [year#34,country#35,product#36,sales#37]\r\n     MetastoreRelation default, sales, None\r\n```\r\n\r\n*\tQuery planning: In the process of query planning, we simple generate the physical Window operator based on the logical Window operator. Then, to prepare the executedPlan, the EnsureRequirements rule will add Exchange and Sort operators if necessary. The EnsureRequirements rule will analyze the data properties and try to not add unnecessary shuffle and sort. The physical plan for the above example query is shown below.\r\n\r\n```\r\n== Physical Plan ==\r\nSort [year#34 ASC,country#35 ASC,product#36 ASC], true\r\n Exchange (RangePartitioning [year#34 ASC,country#35 ASC,product#36 ASC], 200), []\r\n  Window [year#34,country#35,product#36,sales#37,avg_product#27], [HiveWindowFunction#org.apache.hadoop.hive.ql.udf.generic.GenericUDAFSum(sales#37) WindowSpecDefinition [country#35], [], ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS sum_country#28], WindowSpecDefinition [country#35], [], ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING\r\n   Exchange (HashPartitioning [country#35], 200), [country#35 ASC]\r\n    Window [year#34,country#35,product#36,sales#37], [HiveWindowFunction#org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage(sales#37) WindowSpecDefinition [product#36], [], ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS avg_product#27], WindowSpecDefinition [product#36], [], ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING\r\n     Exchange (HashPartitioning [product#36], 200), [product#36 ASC]\r\n      HiveTableScan [year#34,country#35,product#36,sales#37], (MetastoreRelation default, sales, None), None\r\n```\r\n\r\n*\tExecution time: At execution time, a physical Window operator buffers all rows in a partition specified in the partition spec of a OVER clause. If necessary, it also maintains a sliding window frame. The current implementation tries to buffer the input parameters of a window function according to the window frame to avoid evaluating a row multiple times. \r\n\r\n# Future work #\r\n\r\nHere are three improvements that are not hard to add:\r\n*\tTaking advantage of the window frame specification to reduce the number of rows buffered in the physical Window operator. For some cases, we only need to buffer the rows appearing in the sliding window. But for other cases, we will not be able to reduce the number of rows buffered (e.g. ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING). \r\n\r\n*\tWhen aRAGEN frame is used, for <value> PRECEDING and <value> FOLLOWING, it will be great if the <value> part is an expression (we can start with Literal). So, when the data type of ORDER BY expression is a FractionalType, we can support FractionalType as the type <value> (<value> still needs to be evaluated as a positive value).\r\n\r\n*\tWhen aRAGEN frame is used, we need to support DateType and TimestampType as the data type of the expression appearing in the order specification. Then, the <value> part of <value> PRECEDING and <value> FOLLOWING can support interval types (once we support them).\r\n\r\nThis is a joint work with @guowei2 and @yhuai \r\nThanks @hbutani @hvanhovell for his comments\r\nThanks @scwf for his comments and unit tests","created_at":"2015-04-21T06:07:23Z","updated_at":"2015-05-05T17:35:13Z","closed_at":null,"merged_at":null,"merge_commit_sha":"59e56fa2d912ab584c36f43c754efe7c7dbad237","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/5604/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/5604/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/spark/issues/5604/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/206d9e9582b8b2b34ca0d77fa3a1101a07b25cb7","head":{"label":"guowei2:windowImplement","ref":"windowImplement","sha":"206d9e9582b8b2b34ca0d77fa3a1101a07b25cb7","user":{"login":"guowei2","id":6811813,"avatar_url":"https://avatars.githubusercontent.com/u/6811813?v=3","gravatar_id":"","url":"https://api.github.com/users/guowei2","html_url":"https://github.com/guowei2","followers_url":"https://api.github.com/users/guowei2/followers","following_url":"https://api.github.com/users/guowei2/following{/other_user}","gists_url":"https://api.github.com/users/guowei2/gists{/gist_id}","starred_url":"https://api.github.com/users/guowei2/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/guowei2/subscriptions","organizations_url":"https://api.github.com/users/guowei2/orgs","repos_url":"https://api.github.com/users/guowei2/repos","events_url":"https://api.github.com/users/guowei2/events{/privacy}","received_events_url":"https://api.github.com/users/guowei2/received_events","type":"User","site_admin":false},"repo":{"id":23093621,"name":"spark","full_name":"guowei2/spark","owner":{"login":"guowei2","id":6811813,"avatar_url":"https://avatars.githubusercontent.com/u/6811813?v=3","gravatar_id":"","url":"https://api.github.com/users/guowei2","html_url":"https://github.com/guowei2","followers_url":"https://api.github.com/users/guowei2/followers","following_url":"https://api.github.com/users/guowei2/following{/other_user}","gists_url":"https://api.github.com/users/guowei2/gists{/gist_id}","starred_url":"https://api.github.com/users/guowei2/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/guowei2/subscriptions","organizations_url":"https://api.github.com/users/guowei2/orgs","repos_url":"https://api.github.com/users/guowei2/repos","events_url":"https://api.github.com/users/guowei2/events{/privacy}","received_events_url":"https://api.github.com/users/guowei2/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/guowei2/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/guowei2/spark","forks_url":"https://api.github.com/repos/guowei2/spark/forks","keys_url":"https://api.github.com/repos/guowei2/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/guowei2/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/guowei2/spark/teams","hooks_url":"https://api.github.com/repos/guowei2/spark/hooks","issue_events_url":"https://api.github.com/repos/guowei2/spark/issues/events{/number}","events_url":"https://api.github.com/repos/guowei2/spark/events","assignees_url":"https://api.github.com/repos/guowei2/spark/assignees{/user}","branches_url":"https://api.github.com/repos/guowei2/spark/branches{/branch}","tags_url":"https://api.github.com/repos/guowei2/spark/tags","blobs_url":"https://api.github.com/repos/guowei2/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/guowei2/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/guowei2/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/guowei2/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/guowei2/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/guowei2/spark/languages","stargazers_url":"https://api.github.com/repos/guowei2/spark/stargazers","contributors_url":"https://api.github.com/repos/guowei2/spark/contributors","subscribers_url":"https://api.github.com/repos/guowei2/spark/subscribers","subscription_url":"https://api.github.com/repos/guowei2/spark/subscription","commits_url":"https://api.github.com/repos/guowei2/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/guowei2/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/guowei2/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/guowei2/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/guowei2/spark/contents/{+path}","compare_url":"https://api.github.com/repos/guowei2/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/guowei2/spark/merges","archive_url":"https://api.github.com/repos/guowei2/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/guowei2/spark/downloads","issues_url":"https://api.github.com/repos/guowei2/spark/issues{/number}","pulls_url":"https://api.github.com/repos/guowei2/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/guowei2/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/guowei2/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/guowei2/spark/labels{/name}","releases_url":"https://api.github.com/repos/guowei2/spark/releases{/id}","created_at":"2014-08-19T02:12:50Z","updated_at":"2015-05-05T05:28:15Z","pushed_at":"2015-05-05T05:28:10Z","git_url":"git://github.com/guowei2/spark.git","ssh_url":"git@github.com:guowei2/spark.git","clone_url":"https://github.com/guowei2/spark.git","svn_url":"https://github.com/guowei2/spark","homepage":null,"size":101961,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"f32e69ecc333867fc966f65cd0aeaeddd43e0945","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"name":"spark","full_name":"apache/spark","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","created_at":"2014-02-25T08:00:08Z","updated_at":"2015-05-05T16:44:08Z","pushed_at":"2015-05-05T17:30:57Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":1612414,"stargazers_count":3861,"watchers_count":3861,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":3359,"mirror_url":"git://git.apache.org/spark.git","open_issues_count":298,"forks":3359,"open_issues":298,"watchers":3861,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/5604"},"html":{"href":"https://github.com/apache/spark/pull/5604"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/5604"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/5604/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/5604/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/5604/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/206d9e9582b8b2b34ca0d77fa3a1101a07b25cb7"}}}}