{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/30461540","id":30461540,"diff_hunk":"@@ -0,0 +1,144 @@\n+/*\n+ * Licensed to the Apache Software Foundation (ASF) under one or more\n+ * contributor license agreements.  See the NOTICE file distributed with\n+ * this work for additional information regarding copyright ownership.\n+ * The ASF licenses this file to You under the Apache License, Version 2.0\n+ * (the \"License\"); you may not use this file except in compliance with\n+ * the License.  You may obtain a copy of the License at\n+ *\n+ *    http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+\n+package org.apache.spark.sql.hive.orc\n+\n+import org.apache.hadoop.hive.common.`type`.{HiveChar, HiveDecimal, HiveVarchar}\n+import org.apache.hadoop.hive.ql.io.sarg.SearchArgument\n+import org.apache.hadoop.hive.ql.io.sarg.SearchArgument.Builder\n+import org.apache.hadoop.hive.serde2.io.DateWritable\n+\n+import org.apache.spark.Logging\n+import org.apache.spark.sql.sources._\n+\n+/**\n+ * It may be optimized by push down partial filters. But we are conservative here.\n+ * Because if some filters fail to be parsed, the tree may be corrupted,\n+ * and cannot be used anymore.\n+ */\n+private[orc] object OrcFilters extends Logging {\n+  def createFilter(expr: Array[Filter]): Option[SearchArgument] = {\n+    expr.reduceOption(And).flatMap { conjunction =>\n+      val builder = SearchArgument.FACTORY.newBuilder()\n+      buildSearchArgument(conjunction, builder).map(_.build())\n+    }\n+  }\n+\n+  private def buildSearchArgument(expression: Filter, builder: Builder): Option[Builder] = {\n+    def newBuilder = SearchArgument.FACTORY.newBuilder()\n+\n+    def isSearchableLiteral(value: Any) = value match {\n+      // These are types recognized by the `SearchArgumentImpl.BuilderImpl.boxLiteral()` method.\n+      case _: String | _: Long | _: Double | _: DateWritable | _: HiveDecimal | _: HiveChar |\n+           _: HiveVarchar | _: Byte | _: Short | _: Integer | _: Float => true\n+      case _ => false\n+    }\n+\n+    // lian: I probably missed something here, and had to end up with a pretty weird double-checking\n+    // pattern when converting `And`/`Or`/`Not` filters.\n+    //\n+    // The annoying part is that, `SearchArgument` builder methods like `startAnd()` `startOr()`,\n+    // and `startNot()` mutate internal state of the builder instance.  This forces us to translate\n+    // all convertible filters with a single builder instance. However, before actually converting a\n+    // filter, we've no idea whether it can be recognized by ORC or not. Thus, when an inconvertible\n+    // filter is found, we may already end up with a builder whose internal state is inconsistent.\n+    //\n+    // For example, to convert an `And` filter with builder `b`, we call `b.startAnd()` first, and\n+    // then try to convert its children.  Say we convert `left` child successfully, but find that\n+    // `right` child is inconvertible.  Alas, `b.startAnd()` call can't be rolled back, and `b` is\n+    // inconsistent now.\n+    //\n+    // The workaround employed here is that, for `And`/`Or`/`Not`, we first try to convert their\n+    // children with brand new builders, and only do the actual conversion with the right builder\n+    // instance when the children are proven to be convertible.\n+    //\n+    // P.S.: Hive seems to use `SearchArgument` together with `ExprNodeGenericFuncDesc` only.","path":"sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFilters.scala","position":69,"original_position":69,"commit_id":"d4afeed86f2094df8cb6ba509f5a2da22c3bf02b","original_commit_id":"4bc937fa37c2674c007726f6c9bb25911378049f","user":{"login":"liancheng","id":230655,"avatar_url":"https://avatars.githubusercontent.com/u/230655?v=3","gravatar_id":"","url":"https://api.github.com/users/liancheng","html_url":"https://github.com/liancheng","followers_url":"https://api.github.com/users/liancheng/followers","following_url":"https://api.github.com/users/liancheng/following{/other_user}","gists_url":"https://api.github.com/users/liancheng/gists{/gist_id}","starred_url":"https://api.github.com/users/liancheng/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/liancheng/subscriptions","organizations_url":"https://api.github.com/users/liancheng/orgs","repos_url":"https://api.github.com/users/liancheng/repos","events_url":"https://api.github.com/users/liancheng/events{/privacy}","received_events_url":"https://api.github.com/users/liancheng/received_events","type":"User","site_admin":false},"body":"@scwf BTW, with the help of the newly introduced isolated classloader mechanism, Spark SQL can always depend on the most recent version of Hive. At the meanwhile, users can specify arbitrary Hive metastore version to use. So even if this API changes across Hive versions, we don't need shim code to ensure compatibility.","created_at":"2015-05-16T14:35:18Z","updated_at":"2015-05-16T14:35:18Z","html_url":"https://github.com/apache/spark/pull/6194#discussion_r30461540","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/6194","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/30461540"},"html":{"href":"https://github.com/apache/spark/pull/6194#discussion_r30461540"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/6194"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/6194","id":35531886,"html_url":"https://github.com/apache/spark/pull/6194","diff_url":"https://github.com/apache/spark/pull/6194.diff","patch_url":"https://github.com/apache/spark/pull/6194.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/6194","number":6194,"state":"open","locked":false,"title":"[SPARK-2883] [SQL] ORC data source for Spark SQL","user":{"login":"liancheng","id":230655,"avatar_url":"https://avatars.githubusercontent.com/u/230655?v=3","gravatar_id":"","url":"https://api.github.com/users/liancheng","html_url":"https://github.com/liancheng","followers_url":"https://api.github.com/users/liancheng/followers","following_url":"https://api.github.com/users/liancheng/following{/other_user}","gists_url":"https://api.github.com/users/liancheng/gists{/gist_id}","starred_url":"https://api.github.com/users/liancheng/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/liancheng/subscriptions","organizations_url":"https://api.github.com/users/liancheng/orgs","repos_url":"https://api.github.com/users/liancheng/repos","events_url":"https://api.github.com/users/liancheng/events{/privacy}","received_events_url":"https://api.github.com/users/liancheng/received_events","type":"User","site_admin":false},"body":"This PR updates PR #6135 authored by @zhzhan from Hortonworks.\r\n\r\n----\r\n\r\nThis PR implements a Spark SQL data source for accessing ORC files.\r\n\r\n> **NOTE**\r\n>\r\n> Although ORC is now an Apache TLP, the codebase is still tightly coupled with Hive.  That's why the new ORC data source is under `org.apache.spark.sql.hive` package, and must be used with `HiveContext`.  However, it doesn't require existing Hive installation to access ORC files.\r\n\r\n## New Features\r\n\r\n1.  Saving/loading ORC files without contacting Hive metastore\r\n\r\n1.  Support for complex data types (i.e. array, map, and struct)\r\n\r\n1.  Aware of common optimizations provided by Spark SQL:\r\n\r\n    - Column pruning\r\n    - Partitioning pruning\r\n    - Filter push-down\r\n\r\n## Future Work\r\n\r\n1.  Schema evolution support\r\n1.  Hive metastore table conversion\r\n\r\n## Acknowledgements\r\n\r\nThis PR also include initial work done by @scwf from Huawei (PR #3753).","created_at":"2015-05-15T17:09:40Z","updated_at":"2015-05-16T14:35:18Z","closed_at":null,"merged_at":null,"merge_commit_sha":"d953cafa364e41562e925047db0407e964e8f120","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/6194/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/6194/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/spark/issues/6194/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/d4afeed86f2094df8cb6ba509f5a2da22c3bf02b","head":{"label":"liancheng:polishing-orc","ref":"polishing-orc","sha":"d4afeed86f2094df8cb6ba509f5a2da22c3bf02b","user":{"login":"liancheng","id":230655,"avatar_url":"https://avatars.githubusercontent.com/u/230655?v=3","gravatar_id":"","url":"https://api.github.com/users/liancheng","html_url":"https://github.com/liancheng","followers_url":"https://api.github.com/users/liancheng/followers","following_url":"https://api.github.com/users/liancheng/following{/other_user}","gists_url":"https://api.github.com/users/liancheng/gists{/gist_id}","starred_url":"https://api.github.com/users/liancheng/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/liancheng/subscriptions","organizations_url":"https://api.github.com/users/liancheng/orgs","repos_url":"https://api.github.com/users/liancheng/repos","events_url":"https://api.github.com/users/liancheng/events{/privacy}","received_events_url":"https://api.github.com/users/liancheng/received_events","type":"User","site_admin":false},"repo":{"id":17232535,"name":"spark","full_name":"liancheng/spark","owner":{"login":"liancheng","id":230655,"avatar_url":"https://avatars.githubusercontent.com/u/230655?v=3","gravatar_id":"","url":"https://api.github.com/users/liancheng","html_url":"https://github.com/liancheng","followers_url":"https://api.github.com/users/liancheng/followers","following_url":"https://api.github.com/users/liancheng/following{/other_user}","gists_url":"https://api.github.com/users/liancheng/gists{/gist_id}","starred_url":"https://api.github.com/users/liancheng/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/liancheng/subscriptions","organizations_url":"https://api.github.com/users/liancheng/orgs","repos_url":"https://api.github.com/users/liancheng/repos","events_url":"https://api.github.com/users/liancheng/events{/privacy}","received_events_url":"https://api.github.com/users/liancheng/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/liancheng/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/liancheng/spark","forks_url":"https://api.github.com/repos/liancheng/spark/forks","keys_url":"https://api.github.com/repos/liancheng/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/liancheng/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/liancheng/spark/teams","hooks_url":"https://api.github.com/repos/liancheng/spark/hooks","issue_events_url":"https://api.github.com/repos/liancheng/spark/issues/events{/number}","events_url":"https://api.github.com/repos/liancheng/spark/events","assignees_url":"https://api.github.com/repos/liancheng/spark/assignees{/user}","branches_url":"https://api.github.com/repos/liancheng/spark/branches{/branch}","tags_url":"https://api.github.com/repos/liancheng/spark/tags","blobs_url":"https://api.github.com/repos/liancheng/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/liancheng/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/liancheng/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/liancheng/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/liancheng/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/liancheng/spark/languages","stargazers_url":"https://api.github.com/repos/liancheng/spark/stargazers","contributors_url":"https://api.github.com/repos/liancheng/spark/contributors","subscribers_url":"https://api.github.com/repos/liancheng/spark/subscribers","subscription_url":"https://api.github.com/repos/liancheng/spark/subscription","commits_url":"https://api.github.com/repos/liancheng/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/liancheng/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/liancheng/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/liancheng/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/liancheng/spark/contents/{+path}","compare_url":"https://api.github.com/repos/liancheng/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/liancheng/spark/merges","archive_url":"https://api.github.com/repos/liancheng/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/liancheng/spark/downloads","issues_url":"https://api.github.com/repos/liancheng/spark/issues{/number}","pulls_url":"https://api.github.com/repos/liancheng/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/liancheng/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/liancheng/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/liancheng/spark/labels{/name}","releases_url":"https://api.github.com/repos/liancheng/spark/releases{/id}","created_at":"2014-02-27T01:39:03Z","updated_at":"2015-03-31T08:56:24Z","pushed_at":"2015-05-16T14:22:46Z","git_url":"git://github.com/liancheng/spark.git","ssh_url":"git@github.com:liancheng/spark.git","clone_url":"https://github.com/liancheng/spark.git","svn_url":"https://github.com/liancheng/spark","homepage":null,"size":103044,"stargazers_count":1,"watchers_count":1,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":1,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"ce6391296a061bc352386080a2ee96bb63fcc4ac","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"name":"spark","full_name":"apache/spark","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","created_at":"2014-02-25T08:00:08Z","updated_at":"2015-05-16T13:36:09Z","pushed_at":"2015-05-16T14:22:50Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":1679180,"stargazers_count":3985,"watchers_count":3985,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":3505,"mirror_url":"git://git.apache.org/spark.git","open_issues_count":316,"forks":3505,"open_issues":316,"watchers":3985,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/6194"},"html":{"href":"https://github.com/apache/spark/pull/6194"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/6194"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/6194/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/6194/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/6194/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/d4afeed86f2094df8cb6ba509f5a2da22c3bf02b"}}}}