{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/24617792","id":24617792,"diff_hunk":"@@ -187,31 +190,49 @@ case class ParquetRelation2\n   @transient private val fs = FileSystem.get(sparkContext.hadoopConfiguration)\n \n   private class MetadataCache {\n+    // `FileStatus` objects of all \"_metadata\" files.\n     private var metadataStatuses: Array[FileStatus] = _\n+\n+    // `FileStatus` objects of all \"_common_metadata\" files.\n     private var commonMetadataStatuses: Array[FileStatus] = _\n+\n+    // Parquet footer cache.\n     private var footers: Map[FileStatus, Footer] = _\n-    private var parquetSchema: StructType = _\n \n+    // `FileStatus` objects of all data files (Parquet part-files).\n     var dataStatuses: Array[FileStatus] = _\n+\n+    // Partition spec of this table, including names, data types, and values of each partition\n+    // column, and paths of each partition.\n     var partitionSpec: PartitionSpec = _\n+\n+    // Schema of the actual Parquet files, without partition columns discovered from partition\n+    // directory paths.\n+    var parquetSchema: StructType = _\n+\n+    // Schema of the whole table, including partition columns.\n     var schema: StructType = _\n-    var dataSchemaIncludesPartitionKeys: Boolean = _\n \n+    // Indicates whether partition columns are also included in Parquet data file schema.  If not,\n+    // we need to fill in partition column values into read rows when scanning the table.\n+    var partitionKeysIncludedInParquetSchema: Boolean = _\n+\n+    /**\n+     * Refreshes `FileStatus`es, footers, partition spec, and table schema.\n+     */\n     def refresh(): Unit = {\n-      val baseStatuses = {\n-        val statuses = paths.distinct.map(p => fs.getFileStatus(fs.makeQualified(new Path(p))))\n-        // Support either reading a collection of raw Parquet part-files, or a collection of folders\n-        // containing Parquet files (e.g. partitioned Parquet table).\n-        assert(statuses.forall(!_.isDir) || statuses.forall(_.isDir))\n-        statuses.toArray\n-      }\n+      // Support either reading a collection of raw Parquet part-files, or a collection of folders\n+      // containing Parquet files (e.g. partitioned Parquet table).\n+      val baseStatuses = paths.distinct.map { p =>\n+        fs.getFileStatus(fs.makeQualified(new Path(p)))\n+      }.toArray\n+      assert(baseStatuses.forall(!_.isDir) || baseStatuses.forall(_.isDir))\n \n       val leaves = baseStatuses.flatMap { f =>","path":"sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala","position":119,"original_position":119,"commit_id":"efcc8d2d0c22a0675e479068c41036256ba73d23","original_commit_id":"efcc8d2d0c22a0675e479068c41036256ba73d23","user":{"login":"yhuai","id":2072857,"avatar_url":"https://avatars.githubusercontent.com/u/2072857?v=3","gravatar_id":"","url":"https://api.github.com/users/yhuai","html_url":"https://github.com/yhuai","followers_url":"https://api.github.com/users/yhuai/followers","following_url":"https://api.github.com/users/yhuai/following{/other_user}","gists_url":"https://api.github.com/users/yhuai/gists{/gist_id}","starred_url":"https://api.github.com/users/yhuai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/yhuai/subscriptions","organizations_url":"https://api.github.com/users/yhuai/orgs","repos_url":"https://api.github.com/users/yhuai/repos","events_url":"https://api.github.com/users/yhuai/events{/privacy}","received_events_url":"https://api.github.com/users/yhuai/received_events","type":"User","site_admin":false},"body":"Maybe `leafSummaryFiles?`","created_at":"2015-02-12T20:37:18Z","updated_at":"2015-02-12T20:37:18Z","html_url":"https://github.com/apache/spark/pull/4563#discussion_r24617792","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/4563","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/24617792"},"html":{"href":"https://github.com/apache/spark/pull/4563#discussion_r24617792"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/4563"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/4563","id":29168054,"html_url":"https://github.com/apache/spark/pull/4563","diff_url":"https://github.com/apache/spark/pull/4563.diff","patch_url":"https://github.com/apache/spark/pull/4563.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/4563","number":4563,"state":"open","locked":false,"title":"[SPARK-4553] [SPARK-5767] [SQL] Wires Parquet data source with the newly introduced write support for data source API","user":{"login":"liancheng","id":230655,"avatar_url":"https://avatars.githubusercontent.com/u/230655?v=3","gravatar_id":"","url":"https://api.github.com/users/liancheng","html_url":"https://github.com/liancheng","followers_url":"https://api.github.com/users/liancheng/followers","following_url":"https://api.github.com/users/liancheng/following{/other_user}","gists_url":"https://api.github.com/users/liancheng/gists{/gist_id}","starred_url":"https://api.github.com/users/liancheng/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/liancheng/subscriptions","organizations_url":"https://api.github.com/users/liancheng/orgs","repos_url":"https://api.github.com/users/liancheng/repos","events_url":"https://api.github.com/users/liancheng/events{/privacy}","received_events_url":"https://api.github.com/users/liancheng/received_events","type":"User","site_admin":false},"body":"This PR migrates the Parquet data source to the new data source write support API.  Now users can also overwriting and appending to existing tables. Notice that inserting into partitioned tables is not supported yet.\r\n\r\nWhen Parquet data source is enabled, insertion to Hive Metastore Parquet tables is also fullfilled by the Parquet data source. This is done by the newly introduced `HiveMetastoreCatalog.ParquetConversions` rule, which is a \"proper\" implementation of the original hacky `HiveStrategies.ParquetConversion`. The latter is still preserved, and can be removed together with the old Parquet support in the future.\r\n\r\nTODO:\r\n\r\n- [ ] Update outdated comments in `newParquet.scala`.\r\n\r\n<!-- Reviewable:start -->\r\n[<img src=\"https://reviewable.io/review_button.png\" height=40 alt=\"Review on Reviewable\"/>](https://reviewable.io/reviews/apache/spark/4563)\r\n<!-- Reviewable:end -->\r\n","created_at":"2015-02-12T10:36:52Z","updated_at":"2015-02-12T20:37:18Z","closed_at":null,"merged_at":null,"merge_commit_sha":"5c08143a781cd4dd6fc07bfbbe9f1d3ffcaf96ec","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/4563/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/4563/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments/{number}","comments_url":"https://api.github.com/repos/apache/spark/issues/4563/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/efcc8d2d0c22a0675e479068c41036256ba73d23","head":{"label":"liancheng:parquet-refining","ref":"parquet-refining","sha":"efcc8d2d0c22a0675e479068c41036256ba73d23","user":{"login":"liancheng","id":230655,"avatar_url":"https://avatars.githubusercontent.com/u/230655?v=3","gravatar_id":"","url":"https://api.github.com/users/liancheng","html_url":"https://github.com/liancheng","followers_url":"https://api.github.com/users/liancheng/followers","following_url":"https://api.github.com/users/liancheng/following{/other_user}","gists_url":"https://api.github.com/users/liancheng/gists{/gist_id}","starred_url":"https://api.github.com/users/liancheng/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/liancheng/subscriptions","organizations_url":"https://api.github.com/users/liancheng/orgs","repos_url":"https://api.github.com/users/liancheng/repos","events_url":"https://api.github.com/users/liancheng/events{/privacy}","received_events_url":"https://api.github.com/users/liancheng/received_events","type":"User","site_admin":false},"repo":{"id":17232535,"name":"spark","full_name":"liancheng/spark","owner":{"login":"liancheng","id":230655,"avatar_url":"https://avatars.githubusercontent.com/u/230655?v=3","gravatar_id":"","url":"https://api.github.com/users/liancheng","html_url":"https://github.com/liancheng","followers_url":"https://api.github.com/users/liancheng/followers","following_url":"https://api.github.com/users/liancheng/following{/other_user}","gists_url":"https://api.github.com/users/liancheng/gists{/gist_id}","starred_url":"https://api.github.com/users/liancheng/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/liancheng/subscriptions","organizations_url":"https://api.github.com/users/liancheng/orgs","repos_url":"https://api.github.com/users/liancheng/repos","events_url":"https://api.github.com/users/liancheng/events{/privacy}","received_events_url":"https://api.github.com/users/liancheng/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/liancheng/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/liancheng/spark","forks_url":"https://api.github.com/repos/liancheng/spark/forks","keys_url":"https://api.github.com/repos/liancheng/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/liancheng/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/liancheng/spark/teams","hooks_url":"https://api.github.com/repos/liancheng/spark/hooks","issue_events_url":"https://api.github.com/repos/liancheng/spark/issues/events{/number}","events_url":"https://api.github.com/repos/liancheng/spark/events","assignees_url":"https://api.github.com/repos/liancheng/spark/assignees{/user}","branches_url":"https://api.github.com/repos/liancheng/spark/branches{/branch}","tags_url":"https://api.github.com/repos/liancheng/spark/tags","blobs_url":"https://api.github.com/repos/liancheng/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/liancheng/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/liancheng/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/liancheng/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/liancheng/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/liancheng/spark/languages","stargazers_url":"https://api.github.com/repos/liancheng/spark/stargazers","contributors_url":"https://api.github.com/repos/liancheng/spark/contributors","subscribers_url":"https://api.github.com/repos/liancheng/spark/subscribers","subscription_url":"https://api.github.com/repos/liancheng/spark/subscription","commits_url":"https://api.github.com/repos/liancheng/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/liancheng/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/liancheng/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/liancheng/spark/issues/comments/{number}","contents_url":"https://api.github.com/repos/liancheng/spark/contents/{+path}","compare_url":"https://api.github.com/repos/liancheng/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/liancheng/spark/merges","archive_url":"https://api.github.com/repos/liancheng/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/liancheng/spark/downloads","issues_url":"https://api.github.com/repos/liancheng/spark/issues{/number}","pulls_url":"https://api.github.com/repos/liancheng/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/liancheng/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/liancheng/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/liancheng/spark/labels{/name}","releases_url":"https://api.github.com/repos/liancheng/spark/releases{/id}","created_at":"2014-02-27T01:39:03Z","updated_at":"2014-10-21T06:04:03Z","pushed_at":"2015-02-12T10:51:20Z","git_url":"git://github.com/liancheng/spark.git","ssh_url":"git@github.com:liancheng/spark.git","clone_url":"https://github.com/liancheng/spark.git","svn_url":"https://github.com/liancheng/spark","homepage":null,"size":88991,"stargazers_count":1,"watchers_count":1,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":1,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"a38e23c30fb5d12f8f46a119d91a0620036e6800","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"name":"spark","full_name":"apache/spark","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments/{number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","created_at":"2014-02-25T08:00:08Z","updated_at":"2015-02-12T20:15:03Z","pushed_at":"2015-02-12T20:15:02Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":1270671,"stargazers_count":2859,"watchers_count":2859,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":2507,"mirror_url":"git://git.apache.org/spark.git","open_issues_count":289,"forks":2507,"open_issues":289,"watchers":2859,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/4563"},"html":{"href":"https://github.com/apache/spark/pull/4563"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/4563"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/4563/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/4563/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/{number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/4563/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/efcc8d2d0c22a0675e479068c41036256ba73d23"}}}}