{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/30457281","id":30457281,"diff_hunk":"@@ -0,0 +1,144 @@\n+/*\n+ * Licensed to the Apache Software Foundation (ASF) under one or more\n+ * contributor license agreements.  See the NOTICE file distributed with\n+ * this work for additional information regarding copyright ownership.\n+ * The ASF licenses this file to You under the Apache License, Version 2.0\n+ * (the \"License\"); you may not use this file except in compliance with\n+ * the License.  You may obtain a copy of the License at\n+ *\n+ *    http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+\n+package org.apache.spark.sql.hive.orc\n+\n+import org.apache.hadoop.hive.common.`type`.{HiveChar, HiveDecimal, HiveVarchar}\n+import org.apache.hadoop.hive.ql.io.sarg.SearchArgument\n+import org.apache.hadoop.hive.ql.io.sarg.SearchArgument.Builder\n+import org.apache.hadoop.hive.serde2.io.DateWritable\n+\n+import org.apache.spark.Logging\n+import org.apache.spark.sql.sources._\n+\n+/**\n+ * It may be optimized by push down partial filters. But we are conservative here.\n+ * Because if some filters fail to be parsed, the tree may be corrupted,\n+ * and cannot be used anymore.\n+ */\n+private[orc] object OrcFilters extends Logging {\n+  def createFilter(expr: Array[Filter]): Option[SearchArgument] = {\n+    expr.reduceOption(And).flatMap { conjunction =>\n+      val builder = SearchArgument.FACTORY.newBuilder()\n+      buildSearchArgument(conjunction, builder).map(_.build())\n+    }\n+  }\n+\n+  private def buildSearchArgument(expression: Filter, builder: Builder): Option[Builder] = {\n+    def newBuilder = SearchArgument.FACTORY.newBuilder()\n+\n+    def isSearchableLiteral(value: Any) = value match {\n+      // These are types recognized by the `SearchArgumentImpl.BuilderImpl.boxLiteral()` method.\n+      case _: String | _: Long | _: Double | _: DateWritable | _: HiveDecimal | _: HiveChar |\n+           _: HiveVarchar | _: Byte | _: Short | _: Integer | _: Float => true\n+      case _ => false\n+    }\n+\n+    // lian: I probably missed something here, and had to end up with a pretty weird double-checking\n+    // pattern when converting `And`/`Or`/`Not` filters.\n+    //\n+    // The annoying part is that, `SearchArgument` builder methods like `startAnd()` `startOr()`,\n+    // and `startNot()` mutate internal state of the builder instance.  This forces us to translate\n+    // all convertible filters with a single builder instance. However, before actually converting a\n+    // filter, we've no idea whether it can be recognized by ORC or not. Thus, when an inconvertible\n+    // filter is found, we may already end up with a builder whose internal state is inconsistent.\n+    //\n+    // For example, to convert an `And` filter with builder `b`, we call `b.startAnd()` first, and\n+    // then try to convert its children.  Say we convert `left` child successfully, but find that\n+    // `right` child is inconvertible.  Alas, `b.startAnd()` call can't be rolled back, and `b` is\n+    // inconsistent now.\n+    //\n+    // The workaround employed here is that, for `And`/`Or`/`Not`, we first try to convert their\n+    // children with brand new builders, and only do the actual conversion with the right builder\n+    // instance when the children are proven to be convertible.\n+    //\n+    // P.S.: Hive seems to use `SearchArgument` together with `ExprNodeGenericFuncDesc` only.\n+    // Usage of builder methods mentioned above can only be found in test code, where all tested\n+    // filters are known to be convertible.\n+\n+    expression match {\n+      case And(left, right) =>\n+        val tryLeft = buildSearchArgument(left, newBuilder)\n+        val tryRight = buildSearchArgument(right, newBuilder)\n+\n+        val conjunction = for {\n+          _ <- tryLeft\n+          _ <- tryRight\n+          lhs <- buildSearchArgument(left, builder.startAnd())\n+          rhs <- buildSearchArgument(right, lhs)\n+        } yield rhs.end()\n+\n+        // For filter `left AND right`, we can still push down `left` even if `right` is not\n+        // convertible, and vice versa.\n+        conjunction\n+          .orElse(tryLeft.flatMap(_ => buildSearchArgument(left, builder)))\n+          .orElse(tryRight.flatMap(_ => buildSearchArgument(right, builder)))\n+\n+      case And(left, right) =>","path":"sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFilters.scala","position":null,"original_position":91,"commit_id":"563ee1a8641156bdc529a26a060fdc877bc84d3f","original_commit_id":"4bc937fa37c2674c007726f6c9bb25911378049f","user":{"login":"liancheng","id":230655,"avatar_url":"https://avatars.githubusercontent.com/u/230655?v=3","gravatar_id":"","url":"https://api.github.com/users/liancheng","html_url":"https://github.com/liancheng","followers_url":"https://api.github.com/users/liancheng/followers","following_url":"https://api.github.com/users/liancheng/following{/other_user}","gists_url":"https://api.github.com/users/liancheng/gists{/gist_id}","starred_url":"https://api.github.com/users/liancheng/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/liancheng/subscriptions","organizations_url":"https://api.github.com/users/liancheng/orgs","repos_url":"https://api.github.com/users/liancheng/repos","events_url":"https://api.github.com/users/liancheng/events{/privacy}","received_events_url":"https://api.github.com/users/liancheng/received_events","type":"User","site_admin":false},"body":"Oops, thanks!","created_at":"2015-05-16T01:57:13Z","updated_at":"2015-05-16T01:57:13Z","html_url":"https://github.com/apache/spark/pull/6194#discussion_r30457281","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/6194","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/30457281"},"html":{"href":"https://github.com/apache/spark/pull/6194#discussion_r30457281"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/6194"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/6194","id":35531886,"html_url":"https://github.com/apache/spark/pull/6194","diff_url":"https://github.com/apache/spark/pull/6194.diff","patch_url":"https://github.com/apache/spark/pull/6194.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/6194","number":6194,"state":"open","locked":false,"title":"[SPARK-2883] [SQL] ORC data source for Spark SQL","user":{"login":"liancheng","id":230655,"avatar_url":"https://avatars.githubusercontent.com/u/230655?v=3","gravatar_id":"","url":"https://api.github.com/users/liancheng","html_url":"https://github.com/liancheng","followers_url":"https://api.github.com/users/liancheng/followers","following_url":"https://api.github.com/users/liancheng/following{/other_user}","gists_url":"https://api.github.com/users/liancheng/gists{/gist_id}","starred_url":"https://api.github.com/users/liancheng/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/liancheng/subscriptions","organizations_url":"https://api.github.com/users/liancheng/orgs","repos_url":"https://api.github.com/users/liancheng/repos","events_url":"https://api.github.com/users/liancheng/events{/privacy}","received_events_url":"https://api.github.com/users/liancheng/received_events","type":"User","site_admin":false},"body":"This PR is an update of #6135 authored by @zhzhan from Hortonworks.\r\n\r\n----\r\n\r\nThis PR implements a Spark SQL data source for accessing ORC files.\r\n\r\n> **NOTE**\r\n>\r\n> Although ORC is now an Apache TLP, the codebase is still tightly coupled with Hive.  That's why the new ORC data source is under `org.apache.spark.sql.hive` package.\r\n\r\n## New Features\r\n\r\n1.  New save/load methods provided:\r\n\r\n    - `df.saveAsOrcFile()`\r\n\r\n      Used to save the table in ORC format.\r\n\r\n    - `sqlContext.orcFile()`\r\n\r\n      Used to import ORC file as a Spark SQL table.\r\n\r\n\r\n    To enable these two methods, please add the following line to enable corresponding implicit conversions:\r\n\r\n    ```scala\r\n    import org.apache.spark.sql.hive.orc._\r\n    ```\r\n\r\n1.  Support for complex data types (i.e. array, map, and struct)\r\n\r\n1.  Aware of common optimizations provided by Spark SQL:\r\n\r\n    - Column pruning\r\n    - Partitioning pruning\r\n    - Filter push-down\r\n\r\n1.  Saving/loading ORC files without contacting Hive metastore\r\n\r\n1.  The orc file is operated in HiveContext, the only reason is due to package issue, and we don’t want to bring in hive dependency into spark sql. Note that orc operations does not relies on Hive metastore.\r\n\r\n## Future Work\r\n\r\n1.  Schema evolution support\r\n1.  Hive metastore table conversion\r\n\r\n## Acknowledgements\r\n\r\nThis PR also include initial work done by @scwf from Huawei (PR #3753).","created_at":"2015-05-15T17:09:40Z","updated_at":"2015-05-16T01:57:13Z","closed_at":null,"merged_at":null,"merge_commit_sha":"030aca9eb9710ed7a0869fd2d2ce55450c35bc34","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/6194/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/6194/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/spark/issues/6194/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/563ee1a8641156bdc529a26a060fdc877bc84d3f","head":{"label":"liancheng:polishing-orc","ref":"polishing-orc","sha":"563ee1a8641156bdc529a26a060fdc877bc84d3f","user":{"login":"liancheng","id":230655,"avatar_url":"https://avatars.githubusercontent.com/u/230655?v=3","gravatar_id":"","url":"https://api.github.com/users/liancheng","html_url":"https://github.com/liancheng","followers_url":"https://api.github.com/users/liancheng/followers","following_url":"https://api.github.com/users/liancheng/following{/other_user}","gists_url":"https://api.github.com/users/liancheng/gists{/gist_id}","starred_url":"https://api.github.com/users/liancheng/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/liancheng/subscriptions","organizations_url":"https://api.github.com/users/liancheng/orgs","repos_url":"https://api.github.com/users/liancheng/repos","events_url":"https://api.github.com/users/liancheng/events{/privacy}","received_events_url":"https://api.github.com/users/liancheng/received_events","type":"User","site_admin":false},"repo":{"id":17232535,"name":"spark","full_name":"liancheng/spark","owner":{"login":"liancheng","id":230655,"avatar_url":"https://avatars.githubusercontent.com/u/230655?v=3","gravatar_id":"","url":"https://api.github.com/users/liancheng","html_url":"https://github.com/liancheng","followers_url":"https://api.github.com/users/liancheng/followers","following_url":"https://api.github.com/users/liancheng/following{/other_user}","gists_url":"https://api.github.com/users/liancheng/gists{/gist_id}","starred_url":"https://api.github.com/users/liancheng/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/liancheng/subscriptions","organizations_url":"https://api.github.com/users/liancheng/orgs","repos_url":"https://api.github.com/users/liancheng/repos","events_url":"https://api.github.com/users/liancheng/events{/privacy}","received_events_url":"https://api.github.com/users/liancheng/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/liancheng/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/liancheng/spark","forks_url":"https://api.github.com/repos/liancheng/spark/forks","keys_url":"https://api.github.com/repos/liancheng/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/liancheng/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/liancheng/spark/teams","hooks_url":"https://api.github.com/repos/liancheng/spark/hooks","issue_events_url":"https://api.github.com/repos/liancheng/spark/issues/events{/number}","events_url":"https://api.github.com/repos/liancheng/spark/events","assignees_url":"https://api.github.com/repos/liancheng/spark/assignees{/user}","branches_url":"https://api.github.com/repos/liancheng/spark/branches{/branch}","tags_url":"https://api.github.com/repos/liancheng/spark/tags","blobs_url":"https://api.github.com/repos/liancheng/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/liancheng/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/liancheng/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/liancheng/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/liancheng/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/liancheng/spark/languages","stargazers_url":"https://api.github.com/repos/liancheng/spark/stargazers","contributors_url":"https://api.github.com/repos/liancheng/spark/contributors","subscribers_url":"https://api.github.com/repos/liancheng/spark/subscribers","subscription_url":"https://api.github.com/repos/liancheng/spark/subscription","commits_url":"https://api.github.com/repos/liancheng/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/liancheng/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/liancheng/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/liancheng/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/liancheng/spark/contents/{+path}","compare_url":"https://api.github.com/repos/liancheng/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/liancheng/spark/merges","archive_url":"https://api.github.com/repos/liancheng/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/liancheng/spark/downloads","issues_url":"https://api.github.com/repos/liancheng/spark/issues{/number}","pulls_url":"https://api.github.com/repos/liancheng/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/liancheng/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/liancheng/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/liancheng/spark/labels{/name}","releases_url":"https://api.github.com/repos/liancheng/spark/releases{/id}","created_at":"2014-02-27T01:39:03Z","updated_at":"2015-03-31T08:56:24Z","pushed_at":"2015-05-16T01:57:00Z","git_url":"git://github.com/liancheng/spark.git","ssh_url":"git@github.com:liancheng/spark.git","clone_url":"https://github.com/liancheng/spark.git","svn_url":"https://github.com/liancheng/spark","homepage":null,"size":103023,"stargazers_count":1,"watchers_count":1,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":1,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"f96b85ab44b82736363764ea39ee62884007f4a3","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"name":"spark","full_name":"apache/spark","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","created_at":"2014-02-25T08:00:08Z","updated_at":"2015-05-16T00:24:21Z","pushed_at":"2015-05-16T01:57:02Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":1658286,"stargazers_count":3985,"watchers_count":3985,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":3502,"mirror_url":"git://git.apache.org/spark.git","open_issues_count":321,"forks":3502,"open_issues":321,"watchers":3985,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/6194"},"html":{"href":"https://github.com/apache/spark/pull/6194"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/6194"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/6194/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/6194/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/6194/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/563ee1a8641156bdc529a26a060fdc877bc84d3f"}}}}