{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/29639871","id":29639871,"diff_hunk":"@@ -0,0 +1,847 @@\n+/*\n+ * Licensed to the Apache Software Foundation (ASF) under one or more\n+ * contributor license agreements.  See the NOTICE file distributed with\n+ * this work for additional information regarding copyright ownership.\n+ * The ASF licenses this file to You under the Apache License, Version 2.0\n+ * (the \"License\"); you may not use this file except in compliance with\n+ * the License.  You may obtain a copy of the License at\n+ *\n+ *    http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+\n+package org.apache.spark.sql.hive.execution\n+\n+import java.io.File\n+import java.util.{Locale, TimeZone}\n+\n+import org.apache.spark.sql.hive.test.TestHive\n+import org.apache.spark.sql.hive.test.TestHive._\n+import org.apache.spark.util.Utils\n+import org.scalatest.BeforeAndAfter\n+\n+/**\n+ * The test suite for window functions. To actually compare results with Hive,\n+ * every test should be created by `createQueryTest`. Because we are reusing tables\n+ * for different tests and there are a few properties needed to let Hive generate golden\n+ * files, every `createQueryTest` calls should explicitly set `reset` to `false`.\n+ */\n+abstract class HiveWindowFunctionQueryBaseSuite extends HiveComparisonTest with BeforeAndAfter {\n+  private val originalTimeZone = TimeZone.getDefault\n+  private val originalLocale = Locale.getDefault\n+  private val testTempDir = Utils.createTempDir()\n+  import org.apache.spark.sql.hive.test.TestHive.implicits._\n+\n+  override def beforeAll() {\n+    TestHive.cacheTables = true\n+    // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)\n+    TimeZone.setDefault(TimeZone.getTimeZone(\"America/Los_Angeles\"))\n+    // Add Locale setting\n+    Locale.setDefault(Locale.US)\n+\n+    // Create the table used in windowing.q\n+    sql(\"DROP TABLE IF EXISTS part\")\n+    sql(\n+      \"\"\"\n+        |CREATE TABLE part(\n+        |  p_partkey INT,\n+        |  p_name STRING,\n+        |  p_mfgr STRING,\n+        |  p_brand STRING,\n+        |  p_type STRING,\n+        |  p_size INT,\n+        |  p_container STRING,\n+        |  p_retailprice DOUBLE,\n+        |  p_comment STRING)\n+      \"\"\".stripMargin)\n+    val testData1 = TestHive.getHiveFile(\"data/files/part_tiny.txt\").getCanonicalPath\n+    sql(\n+      s\"\"\"\n+        |LOAD DATA LOCAL INPATH '$testData1' overwrite into table part\n+      \"\"\".stripMargin)\n+\n+    sql(\"DROP TABLE IF EXISTS over1k\")\n+    sql(\n+      \"\"\"\n+        |create table over1k(\n+        |  t tinyint,\n+        |  si smallint,\n+        |  i int,\n+        |  b bigint,\n+        |  f float,\n+        |  d double,\n+        |  bo boolean,\n+        |  s string,\n+        |  ts timestamp,\n+        |  dec decimal(4,2),\n+        |  bin binary)\n+        |row format delimited\n+        |fields terminated by '|'\n+      \"\"\".stripMargin)\n+    val testData2 = TestHive.getHiveFile(\"data/files/over1k\").getCanonicalPath\n+    sql(\n+      s\"\"\"\n+        |LOAD DATA LOCAL INPATH '$testData2' overwrite into table over1k\n+      \"\"\".stripMargin)\n+\n+    // The following settings are used for generating golden files with Hive.\n+    // We have to use kryo to correctly let Hive serialize plans with window functions.\n+    // This is used to generate golden files.\n+    sql(\"set hive.plan.serialization.format=kryo\")\n+    // Explicitly set fs to local fs.\n+    sql(s\"set fs.default.name=file://$testTempDir/\")\n+    // Ask Hive to run jobs in-process as a single map and reduce task.\n+    sql(\"set mapred.job.tracker=local\")\n+  }\n+\n+  override def afterAll() {\n+    TestHive.cacheTables = false\n+    TimeZone.setDefault(originalTimeZone)\n+    Locale.setDefault(originalLocale)\n+    TestHive.reset()\n+  }\n+\n+  /////////////////////////////////////////////////////////////////////////////\n+  // Tests based on windowing_multipartitioning.q\n+  // Results of the original query file are not deterministic.\n+  /////////////////////////////////////////////////////////////////////////////\n+  createQueryTest(\"windowing_multipartitioning.q (deterministic) 1\",\n+    s\"\"\"\n+      |select s,\n+      |rank() over (partition by s order by si) r,\n+      |sum(b) over (partition by s order by si) sum\n+      |from over1k\n+      |order by s, r, sum;\n+    \"\"\".stripMargin, reset = false)\n+\n+  /* timestamp comparison issue with Hive?\n+  createQueryTest(\"windowing_multipartitioning.q (deterministic) 2\",\n+    s\"\"\"\n+      |select s,\n+      |rank() over (partition by s order by dec desc) r,\n+      |sum(b) over (partition by s order by ts desc) as sum\n+      |from over1k\n+      |where s = 'tom allen' or s = 'bob steinbeck'\n+      |order by s, r, sum;\n+     \"\"\".stripMargin, reset = false)\n+  */\n+\n+  createQueryTest(\"windowing_multipartitioning.q (deterministic) 3\",\n+    s\"\"\"\n+      |select s, sum(i) over (partition by s), sum(f) over (partition by si)\n+      |from over1k where s = 'tom allen' or s = 'bob steinbeck';\n+     \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing_multipartitioning.q (deterministic) 4\",\n+    s\"\"\"\n+      |select s, rank() over (partition by s order by bo),\n+      |rank() over (partition by si order by bin desc) from over1k\n+      |where s = 'tom allen' or s = 'bob steinbeck';\n+     \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing_multipartitioning.q (deterministic) 5\",\n+    s\"\"\"\n+      |select s, sum(f) over (partition by i), row_number() over (order by f)\n+      |from over1k where s = 'tom allen' or s = 'bob steinbeck';\n+     \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing_multipartitioning.q (deterministic) 6\",\n+    s\"\"\"\n+      |select s, rank() over w1,\n+      |rank() over w2\n+      |from over1k\n+      |where s = 'tom allen' or s = 'bob steinbeck'\n+      |window\n+      |w1 as (partition by s order by dec),\n+      |w2 as (partition by si order by f) ;\n+     \"\"\".stripMargin, reset = false)\n+\n+  /////////////////////////////////////////////////////////////////////////////\n+  // Tests based on windowing_navfn.q\n+  // Results of the original query file are not deterministic.\n+  // Also, the original query of\n+  // select i, lead(s) over (partition by bin order by d,i desc) from over1k ;\n+  /////////////////////////////////////////////////////////////////////////////\n+  createQueryTest(\"windowing_navfn.q (deterministic)\",\n+    s\"\"\"\n+      |select s, row_number() over (partition by d order by dec) rn from over1k\n+      |order by s, rn desc;\n+      |select i, lead(s) over (partition by cast(bin as string) order by d,i desc) as l\n+      |from over1k\n+      |order by i desc, l;\n+      |select i, lag(dec) over (partition by i order by s,i,dec) l from over1k\n+      |order by i, l;\n+      |select s, last_value(t) over (partition by d order by f) l from over1k\n+      |order by s, l;\n+      |select s, first_value(s) over (partition by bo order by s) f from over1k\n+      |order by s, f;\n+      |select t, s, i, last_value(i) over (partition by t order by s)\n+      |from over1k where (s = 'oscar allen' or s = 'oscar carson') and t = 10;\n+     \"\"\".stripMargin, reset = false)\n+\n+  /////////////////////////////////////////////////////////////////////////////\n+  // Tests based on windowing_ntile.q\n+  // Results of the original query file are not deterministic.\n+  /////////////////////////////////////////////////////////////////////////////\n+  createQueryTest(\"windowing_ntile.q (deterministic)\",\n+    s\"\"\"\n+      |select i, ntile(10) over (partition by s order by i) n from over1k\n+      |order by i, n;\n+      |select s, ntile(100) over (partition by i order by s) n from over1k\n+      |order by s, n;\n+      |select f, ntile(4) over (partition by d order by f) n from over1k\n+      |order by f, n;\n+      |select d, ntile(1000) over (partition by dec order by d) n from over1k\n+      |order by d, n;\n+     \"\"\".stripMargin, reset = false)\n+\n+  /////////////////////////////////////////////////////////////////////////////\n+  // Tests based on windowing_udaf.q\n+  // Results of the original query file are not deterministic.\n+  /////////////////////////////////////////////////////////////////////////////\n+  createQueryTest(\"windowing_udaf.q (deterministic)\",\n+    s\"\"\"\n+      |select s, min(i) over (partition by s) m from over1k\n+      |order by s, m;\n+      |select s, avg(f) over (partition by si order by s) a from over1k\n+      |order by s, a;\n+      |select s, avg(i) over (partition by t, b order by s) a from over1k\n+      |order by s, a;\n+      |select max(i) over w m from over1k\n+      |order by m window w as (partition by f) ;\n+      |select s, avg(d) over (partition by t order by f) a from over1k\n+      |order by s, a;\n+     \"\"\".stripMargin, reset = false)\n+\n+  /////////////////////////////////////////////////////////////////////////////\n+  // Tests based on windowing_windowspec.q\n+  // Results of the original query file are not deterministic.\n+  /////////////////////////////////////////////////////////////////////////////\n+  createQueryTest(\"windowing_windowspec.q (deterministic)\",\n+    s\"\"\"\n+      |select s, sum(b) over (partition by i order by s,b rows unbounded preceding) as sum\n+      |from over1k order by s, sum;\n+      |select s, sum(f) over (partition by d order by s,f rows unbounded preceding) as sum\n+      |from over1k order by s, sum;\n+      |select s, sum(f) over\n+      |(partition by ts order by f range between current row and unbounded following) as sum\n+      |from over1k order by s, sum;\n+      |select s, avg(f)\n+      |over (partition by ts order by s,f rows between current row and 5 following) avg\n+      |from over1k order by s, avg;\n+      |select s, avg(d) over\n+      |(partition by t order by s,d desc rows between 5 preceding and 5 following) avg\n+      |from over1k order by s, avg;\n+      |select s, sum(i) over(partition by ts order by s) sum from over1k\n+      |order by s, sum;\n+      |select f, sum(f) over\n+      |(partition by ts order by f range between unbounded preceding and current row) sum\n+      |from over1k order by f, sum;\n+      |select s, i, round(avg(d) over (partition by s order by i) / 10.0 , 2) avg\n+      |from over1k order by s, i, avg;\n+      |select s, i, round((avg(d) over  w1 + 10.0) - (avg(d) over w1 - 10.0),2) avg\n+      |from over1k\n+      |order by s, i, avg window w1 as (partition by s order by i);\n+     \"\"\".stripMargin, reset = false)\n+\n+  /////////////////////////////////////////////////////////////////////////////\n+  // Tests based on windowing_rank.q\n+  // Results of the original query file are not deterministic.\n+  /////////////////////////////////////////////////////////////////////////////\n+  createQueryTest(\"windowing_rank.q (deterministic) 1\",\n+    s\"\"\"\n+      |select s, rank() over (partition by f order by t) r from over1k order by s, r;\n+      |select s, dense_rank() over (partition by ts order by i,s desc) as r from over1k\n+      |order by s desc, r desc;\n+      |select s, cume_dist() over (partition by bo order by b,s) cd from over1k\n+      |order by s, cd;\n+      |select s, percent_rank() over (partition by dec order by f) r from over1k\n+      |order by s desc, r desc;\n+     \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing_rank.q (deterministic) 2\",\n+    s\"\"\"\n+      |select ts, dec, rnk\n+      |from\n+      |  (select ts, dec,\n+      |          rank() over (partition by ts order by dec)  as rnk\n+      |          from\n+      |            (select other.ts, other.dec\n+      |             from over1k other\n+      |             join over1k on (other.b = over1k.b)\n+      |            ) joined\n+      |  ) ranked\n+      |where rnk =  1\n+      |order by ts, dec, rnk;\n+     \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing_rank.q (deterministic) 3\",\n+    s\"\"\"\n+      |select ts, dec, rnk\n+      |from\n+      |  (select ts, dec,\n+      |          rank() over (partition by ts order by dec)  as rnk\n+      |          from\n+      |            (select other.ts, other.dec\n+      |             from over1k other\n+      |             join over1k on (other.b = over1k.b)\n+      |            ) joined\n+      |  ) ranked\n+      |where dec = 89.5\n+      |order by ts, dec, rnk;\n+     \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing_rank.q (deterministic) 4\",\n+    s\"\"\"\n+      |select ts, dec, rnk\n+      |from\n+      |  (select ts, dec,\n+      |          rank() over (partition by ts order by dec)  as rnk\n+      |          from\n+      |            (select other.ts, other.dec\n+      |             from over1k other\n+      |             join over1k on (other.b = over1k.b)\n+      |             where other.t < 10\n+      |            ) joined\n+      |  ) ranked\n+      |where rnk = 1\n+      |order by ts, dec, rnk;\n+     \"\"\".stripMargin, reset = false)\n+\n+  /////////////////////////////////////////////////////////////////////////////\n+  // Tests from windowing.q\n+  // We port tests in windowing.q to here because this query file contains too\n+  // many tests and the syntax of test \"-- 7. testJoinWithWindowingAndPTF\"\n+  // is not supported right now.\n+  /////////////////////////////////////////////////////////////////////////////\n+  createQueryTest(\"windowing.q -- 1. testWindowing\",\n+    s\"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |rank() over(distribute by p_mfgr sort by p_name) as r,\n+      |dense_rank() over(distribute by p_mfgr sort by p_name) as dr,\n+      |sum(p_retailprice) over\n+      |(distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) as s1\n+      |from part\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 2. testGroupByWithPartitioning\",\n+    s\"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |min(p_retailprice),\n+      |rank() over(distribute by p_mfgr sort by p_name)as r,\n+      |dense_rank() over(distribute by p_mfgr sort by p_name) as dr,\n+      |p_size, p_size - lag(p_size,1,p_size) over(distribute by p_mfgr sort by p_name) as deltaSz\n+      |from part\n+      |group by p_mfgr, p_name, p_size\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 3. testGroupByHavingWithSWQ\",\n+    s\"\"\"\n+      |select p_mfgr, p_name, p_size, min(p_retailprice),\n+      |rank() over(distribute by p_mfgr sort by p_name) as r,\n+      |dense_rank() over(distribute by p_mfgr sort by p_name) as dr,\n+      |p_size, p_size - lag(p_size,1,p_size) over(distribute by p_mfgr sort by p_name) as deltaSz\n+      |from part\n+      |group by p_mfgr, p_name, p_size\n+      |having p_size > 0\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 4. testCount\",\n+    s\"\"\"\n+      |select p_mfgr, p_name,\n+      |count(p_size) over(distribute by p_mfgr sort by p_name) as cd\n+      |from part\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 5. testCountWithWindowingUDAF\",\n+    s\"\"\"\n+      |select p_mfgr, p_name,\n+      |rank() over(distribute by p_mfgr sort by p_name) as r,\n+      |dense_rank() over(distribute by p_mfgr sort by p_name) as dr,\n+      |count(p_size) over(distribute by p_mfgr sort by p_name) as cd,\n+      |p_retailprice, sum(p_retailprice) over (distribute by p_mfgr sort by p_name\n+      |                                  rows between unbounded preceding and current row) as s1,\n+      |p_size, p_size - lag(p_size,1,p_size) over(distribute by p_mfgr sort by p_name) as deltaSz\n+      |from part\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 6. testCountInSubQ\",\n+    s\"\"\"\n+      |select sub1.r, sub1.dr, sub1.cd, sub1.s1, sub1.deltaSz\n+      |from (select p_mfgr, p_name,\n+      |rank() over(distribute by p_mfgr sort by p_name) as r,\n+      |dense_rank() over(distribute by p_mfgr sort by p_name) as dr,\n+      |count(p_size) over(distribute by p_mfgr sort by p_name) as cd,\n+      |p_retailprice, sum(p_retailprice) over (distribute by p_mfgr sort by p_name\n+      |                                  rows between unbounded preceding and current row) as s1,\n+      |p_size, p_size - lag(p_size,1,p_size) over(distribute by p_mfgr sort by p_name) as deltaSz\n+      |from part\n+      |) sub1\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 8. testMixedCaseAlias\",\n+    s\"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |rank() over(distribute by p_mfgr sort by p_name, p_size desc) as R\n+      |from part\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 9. testHavingWithWindowingNoGBY\",\n+    s\"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |rank() over(distribute by p_mfgr sort by p_name) as r,\n+      |dense_rank() over(distribute by p_mfgr sort by p_name) as dr,\n+      |sum(p_retailprice) over (distribute by p_mfgr sort by p_name\n+      |                        rows between unbounded preceding and current row)  as s1\n+      |from part\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 10. testHavingWithWindowingCondRankNoGBY\",\n+    s\"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |rank() over(distribute by p_mfgr sort by p_name) as r,\n+      |dense_rank() over(distribute by p_mfgr sort by p_name) as dr,\n+      |sum(p_retailprice) over (distribute by p_mfgr sort by p_name\n+      |                        rows between unbounded preceding and current row) as s1\n+      |from part\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 11. testFirstLast\",\n+    s\"\"\"\n+      |select  p_mfgr,p_name, p_size,\n+      |sum(p_size) over (distribute by p_mfgr sort by p_name\n+      |rows between current row and current row) as s2,\n+      |first_value(p_size) over w1  as f,\n+      |last_value(p_size, false) over w1  as l\n+      |from part\n+      |window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following)\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 12. testFirstLastWithWhere\",\n+    s\"\"\"\n+      |select  p_mfgr,p_name, p_size,\n+      |rank() over(distribute by p_mfgr sort by p_name) as r,\n+      |sum(p_size) over (distribute by p_mfgr sort by p_name\n+      |rows between current row and current row) as s2,\n+      |first_value(p_size) over w1 as f,\n+      |last_value(p_size, false) over w1 as l\n+      |from part\n+      |where p_mfgr = 'Manufacturer#3'\n+      |window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following)\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 13. testSumWindow\",\n+    s\"\"\"\n+      |select  p_mfgr,p_name, p_size,\n+      |sum(p_size) over w1 as s1,\n+      |sum(p_size) over (distribute by p_mfgr  sort by p_name\n+      |rows between current row and current row)  as s2\n+      |from part\n+      |window w1 as (distribute by p_mfgr  sort by p_name rows between 2 preceding and 2 following)\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 14. testNoSortClause\",\n+    s\"\"\"\n+      |select  p_mfgr,p_name, p_size,\n+      |rank() over(distribute by p_mfgr sort by p_name) as r,\n+      |dense_rank() over(distribute by p_mfgr sort by p_name) as dr\n+      |from part\n+      |window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following)\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 15. testExpressions\",\n+    s\"\"\"\n+      |select  p_mfgr,p_name, p_size,\n+      |rank() over(distribute by p_mfgr sort by p_name) as r,\n+      |dense_rank() over(distribute by p_mfgr sort by p_name) as dr,\n+      |cume_dist() over(distribute by p_mfgr sort by p_name) as cud,\n+      |percent_rank() over(distribute by p_mfgr sort by p_name) as pr,\n+      |ntile(3) over(distribute by p_mfgr sort by p_name) as nt,\n+      |count(p_size) over(distribute by p_mfgr sort by p_name) as ca,\n+      |avg(p_size) over(distribute by p_mfgr sort by p_name) as avg,\n+      |stddev(p_size) over(distribute by p_mfgr sort by p_name) as st,\n+      |first_value(p_size % 5) over(distribute by p_mfgr sort by p_name) as fv,\n+      |last_value(p_size) over(distribute by p_mfgr sort by p_name) as lv,\n+      |first_value(p_size) over w1  as fvW1\n+      |from part\n+      |window w1 as (distribute by p_mfgr sort by p_mfgr, p_name\n+      |             rows between 2 preceding and 2 following)\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 16. testMultipleWindows\",\n+    s\"\"\"\n+      |select  p_mfgr,p_name, p_size,\n+      |rank() over(distribute by p_mfgr sort by p_name) as r,\n+      |dense_rank() over(distribute by p_mfgr sort by p_name) as dr,\n+      |cume_dist() over(distribute by p_mfgr sort by p_name) as cud,\n+      |sum(p_size) over (distribute by p_mfgr sort by p_name\n+      |range between unbounded preceding and current row) as s1,\n+      |sum(p_size) over (distribute by p_mfgr sort by p_size\n+      |range between 5 preceding and current row) as s2,\n+      |first_value(p_size) over w1  as fv1\n+      |from part\n+      |window w1 as (distribute by p_mfgr sort by p_mfgr, p_name\n+      |             rows between 2 preceding and 2 following)\n+    \"\"\".stripMargin, reset = false)\n+\n+\n+  createQueryTest(\"windowing.q -- 17. testCountStar\",\n+    s\"\"\"\n+      |select  p_mfgr,p_name, p_size,\n+      |count(*) over(distribute by p_mfgr sort by p_name ) as c,\n+      |count(p_size) over(distribute by p_mfgr sort by p_name) as ca,\n+      |first_value(p_size) over w1  as fvW1\n+      |from part\n+      |window w1 as (distribute by p_mfgr sort by p_mfgr, p_name\n+      |             rows between 2 preceding and 2 following)\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 18. testUDAFs\",\n+    s\"\"\"\n+      |select  p_mfgr,p_name, p_size,\n+      |sum(p_retailprice) over w1 as s,\n+      |min(p_retailprice) over w1 as mi,\n+      |max(p_retailprice) over w1 as ma,\n+      |avg(p_retailprice) over w1 as ag\n+      |from part\n+      |window w1 as (distribute by p_mfgr sort by p_mfgr, p_name\n+      |             rows between 2 preceding and 2 following)\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 19. testUDAFsWithGBY\",\n+    \"\"\"\n+      |select  p_mfgr,p_name, p_size, p_retailprice,\n+      |sum(p_retailprice) over w1 as s,\n+      |min(p_retailprice) as mi ,\n+      |max(p_retailprice) as ma ,\n+      |avg(p_retailprice) over w1 as ag\n+      |from part\n+      |group by p_mfgr,p_name, p_size, p_retailprice\n+      |window w1 as (distribute by p_mfgr sort by p_mfgr, p_name\n+      |             rows between 2 preceding and 2 following);\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 20. testSTATs\",\n+    \"\"\"\n+      |select  p_mfgr,p_name, p_size,\n+      |stddev(p_retailprice) over w1 as sdev,\n+      |stddev_pop(p_retailprice) over w1 as sdev_pop,\n+      |collect_set(p_size) over w1 as uniq_size,\n+      |variance(p_retailprice) over w1 as var,\n+      |corr(p_size, p_retailprice) over w1 as cor,\n+      |covar_pop(p_size, p_retailprice) over w1 as covarp\n+      |from part\n+      |window w1 as (distribute by p_mfgr sort by p_mfgr, p_name\n+      |             rows between 2 preceding and 2 following)\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 21. testDISTs\",\n+    \"\"\"\n+      |select  p_mfgr,p_name, p_size,\n+      |histogram_numeric(p_retailprice, 5) over w1 as hist,\n+      |percentile(p_partkey, 0.5) over w1 as per,\n+      |row_number() over(distribute by p_mfgr sort by p_name) as rn\n+      |from part\n+      |window w1 as (distribute by p_mfgr sort by p_mfgr, p_name\n+      |             rows between 2 preceding and 2 following)\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 24. testLateralViews\",\n+    \"\"\"\n+      |select p_mfgr, p_name,\n+      |lv_col, p_size, sum(p_size) over w1   as s\n+      |from (select p_mfgr, p_name, p_size, array(1,2,3) arr from part) p\n+      |lateral view explode(arr) part_lv as lv_col\n+      |window w1 as (distribute by p_mfgr sort by p_size, lv_col\n+      |             rows between 2 preceding and current row)\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 26. testGroupByHavingWithSWQAndAlias\",\n+    \"\"\"\n+      |select p_mfgr, p_name, p_size, min(p_retailprice) as mi,\n+      |rank() over(distribute by p_mfgr sort by p_name) as r,\n+      |dense_rank() over(distribute by p_mfgr sort by p_name) as dr,\n+      |p_size, p_size - lag(p_size,1,p_size) over(distribute by p_mfgr sort by p_name) as deltaSz\n+      |from part\n+      |group by p_mfgr, p_name, p_size\n+      |having p_size > 0\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 27. testMultipleRangeWindows\",\n+    \"\"\"\n+      |select  p_mfgr,p_name, p_size,\n+      |sum(p_size) over (distribute by p_mfgr sort by p_size\n+      |range between 10 preceding and current row) as s2,\n+      |sum(p_size) over (distribute by p_mfgr sort by p_size\n+      |range between current row and 10 following )  as s1\n+      |from part\n+      |window w1 as (rows between 2 preceding and 2 following)\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 28. testPartOrderInUDAFInvoke\",\n+    \"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |sum(p_size) over (partition by p_mfgr  order by p_name\n+      |rows between 2 preceding and 2 following) as s\n+      |from part\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 29. testPartOrderInWdwDef\",\n+    \"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |sum(p_size) over w1 as s\n+      |from part\n+      |window w1 as (partition by p_mfgr  order by p_name\n+      |             rows between 2 preceding and 2 following)\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 30. testDefaultPartitioningSpecRules\",\n+    \"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |sum(p_size) over w1 as s,\n+      |sum(p_size) over w2 as s2\n+      |from part\n+      |window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following),\n+      |       w2 as (partition by p_mfgr order by p_name)\n+    \"\"\".stripMargin, reset = false)\n+\n+  /* p_name is not a numeric column. What is Hive's semantic?\n+  createQueryTest(\"windowing.q -- 31. testWindowCrossReference\",\n+    \"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |sum(p_size) over w1 as s1,\n+      |sum(p_size) over w2 as s2\n+      |from part\n+      |window w1 as (partition by p_mfgr order by p_name\n+      |             range between 2 preceding and 2 following),\n+      |       w2 as w1\n+    \"\"\".stripMargin, reset = false)\n+  */\n+  /*\n+  createQueryTest(\"windowing.q -- 32. testWindowInheritance\",\n+    \"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |sum(p_size) over w1 as s1,\n+      |sum(p_size) over w2 as s2\n+      |from part\n+      |window w1 as (partition by p_mfgr order by p_name\n+      |             range between 2 preceding and 2 following),\n+      |       w2 as (w1 rows between unbounded preceding and current row)\n+    \"\"\".stripMargin, reset = false)\n+  */\n+\n+  /* p_name is not a numeric column. What is Hive's semantic?\n+  createQueryTest(\"windowing.q -- 33. testWindowForwardReference\",\n+    \"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |sum(p_size) over w1 as s1,\n+      |sum(p_size) over w2 as s2,\n+      |sum(p_size) over w3 as s3\n+      |from part\n+      |window w1 as (distribute by p_mfgr sort by p_name\n+      |             range between 2 preceding and 2 following),\n+      |       w2 as w3,\n+      |       w3 as (distribute by p_mfgr sort by p_name\n+      |             range between unbounded preceding and current row)\n+    \"\"\".stripMargin, reset = false)\n+  */\n+  /*\n+  createQueryTest(\"windowing.q -- 34. testWindowDefinitionPropagation\",\n+    \"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |sum(p_size) over w1 as s1,\n+      |sum(p_size) over w2 as s2,\n+      |sum(p_size) over (w3 rows between 2 preceding and 2 following)  as s3\n+      |from part\n+      |window w1 as (distribute by p_mfgr sort by p_name\n+      |             range between 2 preceding and 2 following),\n+      |       w2 as w3,\n+      |       w3 as (distribute by p_mfgr sort by p_name\n+      |             range between unbounded preceding and current row)\n+    \"\"\".stripMargin, reset = false)\n+  */\n+\n+  /* Seems Hive evaluate SELECT DISTINCT before window functions?\n+  createQueryTest(\"windowing.q -- 35. testDistinctWithWindowing\",\n+    \"\"\"\n+      |select DISTINCT p_mfgr, p_name, p_size,\n+      |sum(p_size) over w1 as s\n+      |from part\n+      |window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following)\n+    \"\"\".stripMargin, reset = false)\n+  */\n+\n+  createQueryTest(\"windowing.q -- 36. testRankWithPartitioning\",\n+    \"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |rank() over (partition by p_mfgr order by p_name )  as r\n+      |from part\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 37. testPartitioningVariousForms\",\n+    \"\"\"\n+      |select p_mfgr,\n+      |round(sum(p_retailprice) over (partition by p_mfgr order by p_mfgr),2) as s1,\n+      |min(p_retailprice) over (partition by p_mfgr) as s2,\n+      |max(p_retailprice) over (distribute by p_mfgr sort by p_mfgr) as s3,\n+      |round(avg(p_retailprice) over (distribute by p_mfgr),2) as s4,\n+      |count(p_retailprice) over (cluster by p_mfgr ) as s5\n+      |from part\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 38. testPartitioningVariousForms2\",\n+    \"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |sum(p_retailprice) over (partition by p_mfgr, p_name order by p_mfgr, p_name\n+      |rows between unbounded preceding and current row) as s1,\n+      |min(p_retailprice) over (distribute by p_mfgr, p_name sort by p_mfgr, p_name\n+      |rows between unbounded preceding and current row) as s2,\n+      |max(p_retailprice) over (partition by p_mfgr, p_name order by p_name) as s3\n+      |from part\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 39. testUDFOnOrderCols\",\n+    \"\"\"\n+      |select p_mfgr, p_type, substr(p_type, 2) as short_ptype,\n+      |rank() over (partition by p_mfgr order by substr(p_type, 2))  as r\n+      |from part\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 40. testNoBetweenForRows\",\n+    \"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |sum(p_retailprice) over (distribute by p_mfgr sort by p_name rows unbounded preceding) as s1\n+      |from part\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 41. testNoBetweenForRange\",\n+    \"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |sum(p_retailprice) over (distribute by p_mfgr sort by p_size range unbounded preceding) as s1\n+      |from part\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 42. testUnboundedFollowingForRows\",\n+    \"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |sum(p_retailprice) over (distribute by p_mfgr sort by p_name\n+      |rows between current row and unbounded following) as s1\n+      |from part\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 43. testUnboundedFollowingForRange\",\n+    \"\"\"\n+      |select p_mfgr, p_name, p_size,\n+      |sum(p_retailprice) over (distribute by p_mfgr sort by p_size\n+      |range between current row and unbounded following) as s1\n+      |from part\n+    \"\"\".stripMargin, reset = false)\n+\n+  createQueryTest(\"windowing.q -- 44. testOverNoPartitionSingleAggregate\",\n+    \"\"\"\n+      |select p_name, p_retailprice,\n+      |round(avg(p_retailprice) over(),2)\n+      |from part\n+      |order by p_name\n+    \"\"\".stripMargin, reset = false)\n+}\n+\n+class HiveWindowFunctionQueryWithoutCodeGenSuite extends HiveWindowFunctionQueryBaseSuite {\n+  var originalCodegenEnabled: Boolean = _\n+  override def beforeAll(): Unit = {\n+    super.beforeAll()\n+    originalCodegenEnabled = conf.codegenEnabled\n+    sql(\"set spark.sql.codegen=false\")\n+  }\n+\n+  override def afterAll(): Unit = {\n+    sql(s\"set spark.sql.codegen=$originalCodegenEnabled\")\n+    super.afterAll()\n+  }\n+}\n+\n+abstract class HiveWindowFunctionQueryFileBaseSuite\n+  extends HiveCompatibilitySuite with BeforeAndAfter {\n+  private val originalTimeZone = TimeZone.getDefault\n+  private val originalLocale = Locale.getDefault\n+  private val testTempDir = Utils.createTempDir()\n+\n+  import org.apache.spark.sql.hive.test.TestHive.implicits._\n+\n+  override def beforeAll() {\n+    TestHive.cacheTables = true\n+    // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)\n+    TimeZone.setDefault(TimeZone.getTimeZone(\"America/Los_Angeles\"))\n+    // Add Locale setting\n+    Locale.setDefault(Locale.US)\n+\n+    // The following settings are used for generating golden files with Hive.\n+    // We have to use kryo to correctly let Hive serialize plans with window functions.\n+    // This is used to generate golden files.\n+    sql(\"set hive.plan.serialization.format=kryo\")","path":"sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveWindowFunctionQuerySuite.scala","position":786,"original_position":786,"commit_id":"206d9e9582b8b2b34ca0d77fa3a1101a07b25cb7","original_commit_id":"206d9e9582b8b2b34ca0d77fa3a1101a07b25cb7","user":{"login":"scwf","id":7018048,"avatar_url":"https://avatars.githubusercontent.com/u/7018048?v=3","gravatar_id":"","url":"https://api.github.com/users/scwf","html_url":"https://github.com/scwf","followers_url":"https://api.github.com/users/scwf/followers","following_url":"https://api.github.com/users/scwf/following{/other_user}","gists_url":"https://api.github.com/users/scwf/gists{/gist_id}","starred_url":"https://api.github.com/users/scwf/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/scwf/subscriptions","organizations_url":"https://api.github.com/users/scwf/orgs","repos_url":"https://api.github.com/users/scwf/repos","events_url":"https://api.github.com/users/scwf/events{/privacy}","received_events_url":"https://api.github.com/users/scwf/received_events","type":"User","site_admin":false},"body":"not related to this PR. \r\nmaybe we should set kryo by default, otherwise there are many tests can not generate golden answer(i have tested it locally)","created_at":"2015-05-05T01:52:12Z","updated_at":"2015-05-05T01:52:12Z","html_url":"https://github.com/apache/spark/pull/5604#discussion_r29639871","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/5604","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/29639871"},"html":{"href":"https://github.com/apache/spark/pull/5604#discussion_r29639871"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/5604"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/5604","id":33731513,"html_url":"https://github.com/apache/spark/pull/5604","diff_url":"https://github.com/apache/spark/pull/5604.diff","patch_url":"https://github.com/apache/spark/pull/5604.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/5604","number":5604,"state":"open","locked":false,"title":"[SPARK-1442][SQL] Window Function Support for Spark SQL","user":{"login":"guowei2","id":6811813,"avatar_url":"https://avatars.githubusercontent.com/u/6811813?v=3","gravatar_id":"","url":"https://api.github.com/users/guowei2","html_url":"https://github.com/guowei2","followers_url":"https://api.github.com/users/guowei2/followers","following_url":"https://api.github.com/users/guowei2/following{/other_user}","gists_url":"https://api.github.com/users/guowei2/gists{/gist_id}","starred_url":"https://api.github.com/users/guowei2/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/guowei2/subscriptions","organizations_url":"https://api.github.com/users/guowei2/orgs","repos_url":"https://api.github.com/users/guowei2/repos","events_url":"https://api.github.com/users/guowei2/events{/privacy}","received_events_url":"https://api.github.com/users/guowei2/received_events","type":"User","site_admin":false},"body":"Window Function Support for Spark SQL","created_at":"2015-04-21T06:07:23Z","updated_at":"2015-05-05T01:52:12Z","closed_at":null,"merged_at":null,"merge_commit_sha":"3e80bd7767c180fee871ace285bca081ecfbcfc3","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/5604/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/5604/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/spark/issues/5604/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/206d9e9582b8b2b34ca0d77fa3a1101a07b25cb7","head":{"label":"guowei2:windowImplement","ref":"windowImplement","sha":"206d9e9582b8b2b34ca0d77fa3a1101a07b25cb7","user":{"login":"guowei2","id":6811813,"avatar_url":"https://avatars.githubusercontent.com/u/6811813?v=3","gravatar_id":"","url":"https://api.github.com/users/guowei2","html_url":"https://github.com/guowei2","followers_url":"https://api.github.com/users/guowei2/followers","following_url":"https://api.github.com/users/guowei2/following{/other_user}","gists_url":"https://api.github.com/users/guowei2/gists{/gist_id}","starred_url":"https://api.github.com/users/guowei2/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/guowei2/subscriptions","organizations_url":"https://api.github.com/users/guowei2/orgs","repos_url":"https://api.github.com/users/guowei2/repos","events_url":"https://api.github.com/users/guowei2/events{/privacy}","received_events_url":"https://api.github.com/users/guowei2/received_events","type":"User","site_admin":false},"repo":{"id":23093621,"name":"spark","full_name":"guowei2/spark","owner":{"login":"guowei2","id":6811813,"avatar_url":"https://avatars.githubusercontent.com/u/6811813?v=3","gravatar_id":"","url":"https://api.github.com/users/guowei2","html_url":"https://github.com/guowei2","followers_url":"https://api.github.com/users/guowei2/followers","following_url":"https://api.github.com/users/guowei2/following{/other_user}","gists_url":"https://api.github.com/users/guowei2/gists{/gist_id}","starred_url":"https://api.github.com/users/guowei2/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/guowei2/subscriptions","organizations_url":"https://api.github.com/users/guowei2/orgs","repos_url":"https://api.github.com/users/guowei2/repos","events_url":"https://api.github.com/users/guowei2/events{/privacy}","received_events_url":"https://api.github.com/users/guowei2/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/guowei2/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/guowei2/spark","forks_url":"https://api.github.com/repos/guowei2/spark/forks","keys_url":"https://api.github.com/repos/guowei2/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/guowei2/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/guowei2/spark/teams","hooks_url":"https://api.github.com/repos/guowei2/spark/hooks","issue_events_url":"https://api.github.com/repos/guowei2/spark/issues/events{/number}","events_url":"https://api.github.com/repos/guowei2/spark/events","assignees_url":"https://api.github.com/repos/guowei2/spark/assignees{/user}","branches_url":"https://api.github.com/repos/guowei2/spark/branches{/branch}","tags_url":"https://api.github.com/repos/guowei2/spark/tags","blobs_url":"https://api.github.com/repos/guowei2/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/guowei2/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/guowei2/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/guowei2/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/guowei2/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/guowei2/spark/languages","stargazers_url":"https://api.github.com/repos/guowei2/spark/stargazers","contributors_url":"https://api.github.com/repos/guowei2/spark/contributors","subscribers_url":"https://api.github.com/repos/guowei2/spark/subscribers","subscription_url":"https://api.github.com/repos/guowei2/spark/subscription","commits_url":"https://api.github.com/repos/guowei2/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/guowei2/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/guowei2/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/guowei2/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/guowei2/spark/contents/{+path}","compare_url":"https://api.github.com/repos/guowei2/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/guowei2/spark/merges","archive_url":"https://api.github.com/repos/guowei2/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/guowei2/spark/downloads","issues_url":"https://api.github.com/repos/guowei2/spark/issues{/number}","pulls_url":"https://api.github.com/repos/guowei2/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/guowei2/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/guowei2/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/guowei2/spark/labels{/name}","releases_url":"https://api.github.com/repos/guowei2/spark/releases{/id}","created_at":"2014-08-19T02:12:50Z","updated_at":"2015-04-24T02:04:17Z","pushed_at":"2015-05-04T22:28:39Z","git_url":"git://github.com/guowei2/spark.git","ssh_url":"git@github.com:guowei2/spark.git","clone_url":"https://github.com/guowei2/spark.git","svn_url":"https://github.com/guowei2/spark","homepage":null,"size":101699,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"f32e69ecc333867fc966f65cd0aeaeddd43e0945","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"name":"spark","full_name":"apache/spark","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","created_at":"2014-02-25T08:00:08Z","updated_at":"2015-05-05T01:29:37Z","pushed_at":"2015-05-05T01:42:38Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":1590090,"stargazers_count":3855,"watchers_count":3855,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":3346,"mirror_url":"git://git.apache.org/spark.git","open_issues_count":308,"forks":3346,"open_issues":308,"watchers":3855,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/5604"},"html":{"href":"https://github.com/apache/spark/pull/5604"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/5604"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/5604/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/5604/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/5604/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/206d9e9582b8b2b34ca0d77fa3a1101a07b25cb7"}}}}