{"action":"created","comment":{"url":"https://api.github.com/repos/apache/incubator-hawq/pulls/comments/44705317","id":44705317,"diff_hunk":"@@ -0,0 +1,226 @@\n+package org.apache.hawq.pxf.api;\n+\n+import org.apache.commons.logging.Log;\n+import org.apache.commons.logging.LogFactory;\n+import org.codehaus.jackson.map.ObjectMapper;\n+\n+import java.io.IOException;\n+\n+/**\n+ * FragmentsStats holds statistics for a given path.\n+ */\n+public class FragmentsStats {\n+\n+    /**\n+     * Default fragment size. Assuming a fragment is equivalent to a block in\n+     * HDFS, we guess a full fragment size is 64MB.\n+     */\n+    public static final long DEFAULT_FRAGMENT_SIZE = 67108864L;\n+\n+    private static Log Log = LogFactory.getLog(FragmentsStats.class);\n+\n+    // number of fragments\n+    private long fragmentsNumber;\n+    // first fragment size\n+    private SizeAndUnit firstFragmentSize;\n+    // total fragments size\n+    private SizeAndUnit totalSize;\n+\n+    /**\n+     * Enum to represent unit (Bytes/MB/GB/TB)\n+     */\n+    public enum SizeUnit {\n+        /**\n+         * Byte\n+         */\n+        B,\n+        /**\n+         * KB\n+         */\n+        KB,\n+        /**\n+         * MB\n+         */\n+        MB,\n+        /**\n+         * GB\n+         */\n+        GB,\n+        /**\n+         * TB\n+         */\n+        TB;\n+    };\n+\n+    /**\n+     * Container for size and unit\n+     */\n+    public class SizeAndUnit {\n+        long size;\n+        SizeUnit unit;\n+\n+        /**\n+         * Default constructor.\n+         */\n+        public SizeAndUnit() {\n+            this.size = 0;\n+            this.unit = SizeUnit.B;\n+        }\n+\n+        /**\n+         * Constructor.\n+         *\n+         * @param size size\n+         * @param unit unit\n+         */\n+        public SizeAndUnit(long size, SizeUnit unit) {\n+            this.size = size;\n+            this.unit = unit;\n+        }\n+\n+        /**\n+         * Returns size.\n+         *\n+         * @return size\n+         */\n+        public long getSize() {\n+            return this.size;\n+        }\n+\n+        /**\n+         * Returns unit (Byte/KB/MB/etc.).\n+         *\n+         * @return unit\n+         */\n+        public SizeUnit getUnit() {\n+            return this.unit;\n+        }\n+\n+        @Override\n+        public String toString() {\n+            return size + \"\" + unit;\n+        }\n+    }\n+\n+    /**\n+     * Constructs an FragmentsStats.\n+     *\n+     * @param fragmentsNumber number of fragments\n+     * @param firstFragmentSize first fragment size (in bytes)\n+     * @param totalSize total size (in bytes)\n+     */\n+    public FragmentsStats(long fragmentsNumber, long firstFragmentSize,\n+                          long totalSize) {\n+        this.setFragmentsNumber(fragmentsNumber);\n+        this.setFirstFragmentSize(firstFragmentSize);\n+        this.setTotalSize(totalSize);\n+    }\n+\n+    /**\n+     * Given a {@link FragmentsStats}, serialize it in JSON to be used as the\n+     * result string for HAWQ. An example result is as follows:\n+     * <code>{\"PXFFragmentsStats\":{\"fragmentsNumber\"\n+     * :3,\"firstFragmentSize\":67108864,\"totalSize\":200000000}}</code>\n+     *\n+     * @param stats the data to be serialized\n+     * @return the result in json format\n+     * @throws IOException if converting to JSON format failed\n+     */\n+    public static String dataToJSON(FragmentsStats stats) throws IOException {\n+        ObjectMapper mapper = new ObjectMapper();\n+        // mapper serializes all members of the class by default\n+        return \"{\\\"PXFFragmentsStats\\\":\" + mapper.writeValueAsString(stats)\n+                + \"}\";\n+    }\n+\n+    /**\n+     * Given a stats structure, convert it to be readable. Intended for\n+     * debugging purposes only.\n+     *\n+     * @param stats the data to be stringify\n+     * @param datapath the data path part of the original URI (e.g., table name,\n+     *            *.csv, etc.)\n+     * @return the stringified data\n+     */\n+    public static String dataToString(FragmentsStats stats, String datapath) {\n+        return \"Statistics information for \\\"\" + datapath + \"\\\" \"\n+                + \" Number of Fragments: \" + stats.fragmentsNumber\n+                + \", first Fragment size: \" + stats.firstFragmentSize\n+                + \", total size: \" + stats.totalSize;\n+    }\n+\n+    /**\n+     * Returns number of fragments for a given data source.\n+     *\n+     * @return number of fragments\n+     */\n+    public long getFragmentsNumber() {\n+        return fragmentsNumber;\n+    }\n+\n+    private void setFragmentsNumber(long fragmentsNumber) {\n+        this.fragmentsNumber = fragmentsNumber;\n+    }\n+\n+    /**\n+     * Returns the size in bytes of the first fragment.\n+     *\n+     * @return first fragment size (in byte)\n+     */\n+    public SizeAndUnit getFirstFragmentSize() {\n+        return firstFragmentSize;\n+    }\n+\n+    private void setFirstFragmentSize(long firstFragmentSize) {\n+        this.firstFragmentSize = setSizeAndUnit(firstFragmentSize);\n+    }\n+\n+    /**\n+     * Returns the total size of a given source. Usually it means the\n+     * aggregation of all its fragments size.\n+     *\n+     * @return total size\n+     */\n+    public SizeAndUnit getTotalSize() {\n+        return totalSize;\n+    }\n+\n+    private void setTotalSize(long totalSize) {\n+        this.totalSize = setSizeAndUnit(totalSize);\n+    }\n+\n+    private SizeAndUnit setSizeAndUnit(long originalSize) {\n+        final long THRESHOLD = Integer.MAX_VALUE / 2;\n+        int orderOfMagnitude = 0;\n+        SizeAndUnit sizeAndUnit = new SizeAndUnit();\n+        sizeAndUnit.size = originalSize;\n+\n+        while (sizeAndUnit.size > THRESHOLD) {\n+            sizeAndUnit.size /= 1024;\n+            orderOfMagnitude++;\n+        }\n+\n+        sizeAndUnit.unit = getSizeUnit(orderOfMagnitude);\n+        return sizeAndUnit;\n+    }\n+\n+    private SizeUnit getSizeUnit(int orderOfMagnitude) {\n+        switch (orderOfMagnitude) {\n+            case 0:\n+                return SizeUnit.B;","path":"pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FragmentsStats.java","position":210,"original_position":210,"commit_id":"49b3e448436c32561af2fc16749ec484216770d6","original_commit_id":"49b3e448436c32561af2fc16749ec484216770d6","user":{"login":"shivzone","id":1966435,"avatar_url":"https://avatars.githubusercontent.com/u/1966435?v=3","gravatar_id":"","url":"https://api.github.com/users/shivzone","html_url":"https://github.com/shivzone","followers_url":"https://api.github.com/users/shivzone/followers","following_url":"https://api.github.com/users/shivzone/following{/other_user}","gists_url":"https://api.github.com/users/shivzone/gists{/gist_id}","starred_url":"https://api.github.com/users/shivzone/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/shivzone/subscriptions","organizations_url":"https://api.github.com/users/shivzone/orgs","repos_url":"https://api.github.com/users/shivzone/repos","events_url":"https://api.github.com/users/shivzone/events{/privacy}","received_events_url":"https://api.github.com/users/shivzone/received_events","type":"User","site_admin":false},"body":"no break statement","created_at":"2015-11-12T19:49:39Z","updated_at":"2015-11-12T19:49:39Z","html_url":"https://github.com/apache/incubator-hawq/pull/92#discussion_r44705317","pull_request_url":"https://api.github.com/repos/apache/incubator-hawq/pulls/92","_links":{"self":{"href":"https://api.github.com/repos/apache/incubator-hawq/pulls/comments/44705317"},"html":{"href":"https://github.com/apache/incubator-hawq/pull/92#discussion_r44705317"},"pull_request":{"href":"https://api.github.com/repos/apache/incubator-hawq/pulls/92"}}},"pull_request":{"url":"https://api.github.com/repos/apache/incubator-hawq/pulls/92","id":50294895,"html_url":"https://github.com/apache/incubator-hawq/pull/92","diff_url":"https://github.com/apache/incubator-hawq/pull/92.diff","patch_url":"https://github.com/apache/incubator-hawq/pull/92.patch","issue_url":"https://api.github.com/repos/apache/incubator-hawq/issues/92","number":92,"state":"open","locked":false,"title":"Analyze HAWQ-44","user":{"login":"hornn","id":11283944,"avatar_url":"https://avatars.githubusercontent.com/u/11283944?v=3","gravatar_id":"","url":"https://api.github.com/users/hornn","html_url":"https://github.com/hornn","followers_url":"https://api.github.com/users/hornn/followers","following_url":"https://api.github.com/users/hornn/following{/other_user}","gists_url":"https://api.github.com/users/hornn/gists{/gist_id}","starred_url":"https://api.github.com/users/hornn/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hornn/subscriptions","organizations_url":"https://api.github.com/users/hornn/orgs","repos_url":"https://api.github.com/users/hornn/repos","events_url":"https://api.github.com/users/hornn/events{/privacy}","received_events_url":"https://api.github.com/users/hornn/received_events","type":"User","site_admin":false},"body":"Advanced statistics for PXF table.\r\n\r\nPXF sample rows are collected into a temporary table, where statistics are derived of them in the same way ANALYZE works for hawq tables.\r\nStatistics are gathered at 3 stages:\r\n1. Getting general statistics - number of fragments, size of data source, size of first fragment\r\n2. Count of first fragment tuples\r\nHAWQ uses these numbers to determine how many tuples are needed, and these parameters are translated to sampling ratio and number of sampled fragments.\r\n3. Sampling the PXF table based on the sampling ratio and number of fragments to be sampled. The returned tuples are saved in a temporary table.\r\n\r\nOn the PXF side, a function has been made to the Fragmenter API, to allow gathering the stats of the first stage. In addition, a mechanism to sample rows on the fly was added to the Bridge.","created_at":"2015-11-10T18:55:39Z","updated_at":"2015-11-12T19:49:40Z","closed_at":null,"merged_at":null,"merge_commit_sha":"5a09a95bcdf900a4b3bd7c9bcaa9dc55e42cdeeb","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/apache/incubator-hawq/pulls/92/commits","review_comments_url":"https://api.github.com/repos/apache/incubator-hawq/pulls/92/comments","review_comment_url":"https://api.github.com/repos/apache/incubator-hawq/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/incubator-hawq/issues/92/comments","statuses_url":"https://api.github.com/repos/apache/incubator-hawq/statuses/49b3e448436c32561af2fc16749ec484216770d6","head":{"label":"hornn:analyze_HAWQ-44","ref":"analyze_HAWQ-44","sha":"49b3e448436c32561af2fc16749ec484216770d6","user":{"login":"hornn","id":11283944,"avatar_url":"https://avatars.githubusercontent.com/u/11283944?v=3","gravatar_id":"","url":"https://api.github.com/users/hornn","html_url":"https://github.com/hornn","followers_url":"https://api.github.com/users/hornn/followers","following_url":"https://api.github.com/users/hornn/following{/other_user}","gists_url":"https://api.github.com/users/hornn/gists{/gist_id}","starred_url":"https://api.github.com/users/hornn/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hornn/subscriptions","organizations_url":"https://api.github.com/users/hornn/orgs","repos_url":"https://api.github.com/users/hornn/repos","events_url":"https://api.github.com/users/hornn/events{/privacy}","received_events_url":"https://api.github.com/users/hornn/received_events","type":"User","site_admin":false},"repo":{"id":44128416,"name":"incubator-hawq","full_name":"hornn/incubator-hawq","owner":{"login":"hornn","id":11283944,"avatar_url":"https://avatars.githubusercontent.com/u/11283944?v=3","gravatar_id":"","url":"https://api.github.com/users/hornn","html_url":"https://github.com/hornn","followers_url":"https://api.github.com/users/hornn/followers","following_url":"https://api.github.com/users/hornn/following{/other_user}","gists_url":"https://api.github.com/users/hornn/gists{/gist_id}","starred_url":"https://api.github.com/users/hornn/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hornn/subscriptions","organizations_url":"https://api.github.com/users/hornn/orgs","repos_url":"https://api.github.com/users/hornn/repos","events_url":"https://api.github.com/users/hornn/events{/privacy}","received_events_url":"https://api.github.com/users/hornn/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/hornn/incubator-hawq","description":"Mirror of Apache HAWQ (Incubating)","fork":true,"url":"https://api.github.com/repos/hornn/incubator-hawq","forks_url":"https://api.github.com/repos/hornn/incubator-hawq/forks","keys_url":"https://api.github.com/repos/hornn/incubator-hawq/keys{/key_id}","collaborators_url":"https://api.github.com/repos/hornn/incubator-hawq/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/hornn/incubator-hawq/teams","hooks_url":"https://api.github.com/repos/hornn/incubator-hawq/hooks","issue_events_url":"https://api.github.com/repos/hornn/incubator-hawq/issues/events{/number}","events_url":"https://api.github.com/repos/hornn/incubator-hawq/events","assignees_url":"https://api.github.com/repos/hornn/incubator-hawq/assignees{/user}","branches_url":"https://api.github.com/repos/hornn/incubator-hawq/branches{/branch}","tags_url":"https://api.github.com/repos/hornn/incubator-hawq/tags","blobs_url":"https://api.github.com/repos/hornn/incubator-hawq/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/hornn/incubator-hawq/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/hornn/incubator-hawq/git/refs{/sha}","trees_url":"https://api.github.com/repos/hornn/incubator-hawq/git/trees{/sha}","statuses_url":"https://api.github.com/repos/hornn/incubator-hawq/statuses/{sha}","languages_url":"https://api.github.com/repos/hornn/incubator-hawq/languages","stargazers_url":"https://api.github.com/repos/hornn/incubator-hawq/stargazers","contributors_url":"https://api.github.com/repos/hornn/incubator-hawq/contributors","subscribers_url":"https://api.github.com/repos/hornn/incubator-hawq/subscribers","subscription_url":"https://api.github.com/repos/hornn/incubator-hawq/subscription","commits_url":"https://api.github.com/repos/hornn/incubator-hawq/commits{/sha}","git_commits_url":"https://api.github.com/repos/hornn/incubator-hawq/git/commits{/sha}","comments_url":"https://api.github.com/repos/hornn/incubator-hawq/comments{/number}","issue_comment_url":"https://api.github.com/repos/hornn/incubator-hawq/issues/comments{/number}","contents_url":"https://api.github.com/repos/hornn/incubator-hawq/contents/{+path}","compare_url":"https://api.github.com/repos/hornn/incubator-hawq/compare/{base}...{head}","merges_url":"https://api.github.com/repos/hornn/incubator-hawq/merges","archive_url":"https://api.github.com/repos/hornn/incubator-hawq/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/hornn/incubator-hawq/downloads","issues_url":"https://api.github.com/repos/hornn/incubator-hawq/issues{/number}","pulls_url":"https://api.github.com/repos/hornn/incubator-hawq/pulls{/number}","milestones_url":"https://api.github.com/repos/hornn/incubator-hawq/milestones{/number}","notifications_url":"https://api.github.com/repos/hornn/incubator-hawq/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/hornn/incubator-hawq/labels{/name}","releases_url":"https://api.github.com/repos/hornn/incubator-hawq/releases{/id}","created_at":"2015-10-12T19:10:51Z","updated_at":"2015-10-12T19:11:20Z","pushed_at":"2015-11-12T01:36:07Z","git_url":"git://github.com/hornn/incubator-hawq.git","ssh_url":"git@github.com:hornn/incubator-hawq.git","clone_url":"https://github.com/hornn/incubator-hawq.git","svn_url":"https://github.com/hornn/incubator-hawq","homepage":null,"size":39326,"stargazers_count":0,"watchers_count":0,"language":"C","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"e24233fa29159c58b50cee86752722a0e8bfe0a9","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":41952293,"name":"incubator-hawq","full_name":"apache/incubator-hawq","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/incubator-hawq","description":"Mirror of Apache HAWQ (Incubating)","fork":false,"url":"https://api.github.com/repos/apache/incubator-hawq","forks_url":"https://api.github.com/repos/apache/incubator-hawq/forks","keys_url":"https://api.github.com/repos/apache/incubator-hawq/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/incubator-hawq/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/incubator-hawq/teams","hooks_url":"https://api.github.com/repos/apache/incubator-hawq/hooks","issue_events_url":"https://api.github.com/repos/apache/incubator-hawq/issues/events{/number}","events_url":"https://api.github.com/repos/apache/incubator-hawq/events","assignees_url":"https://api.github.com/repos/apache/incubator-hawq/assignees{/user}","branches_url":"https://api.github.com/repos/apache/incubator-hawq/branches{/branch}","tags_url":"https://api.github.com/repos/apache/incubator-hawq/tags","blobs_url":"https://api.github.com/repos/apache/incubator-hawq/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/incubator-hawq/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/incubator-hawq/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/incubator-hawq/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/incubator-hawq/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/incubator-hawq/languages","stargazers_url":"https://api.github.com/repos/apache/incubator-hawq/stargazers","contributors_url":"https://api.github.com/repos/apache/incubator-hawq/contributors","subscribers_url":"https://api.github.com/repos/apache/incubator-hawq/subscribers","subscription_url":"https://api.github.com/repos/apache/incubator-hawq/subscription","commits_url":"https://api.github.com/repos/apache/incubator-hawq/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/incubator-hawq/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/incubator-hawq/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/incubator-hawq/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/incubator-hawq/contents/{+path}","compare_url":"https://api.github.com/repos/apache/incubator-hawq/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/incubator-hawq/merges","archive_url":"https://api.github.com/repos/apache/incubator-hawq/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/incubator-hawq/downloads","issues_url":"https://api.github.com/repos/apache/incubator-hawq/issues{/number}","pulls_url":"https://api.github.com/repos/apache/incubator-hawq/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/incubator-hawq/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/incubator-hawq/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/incubator-hawq/labels{/name}","releases_url":"https://api.github.com/repos/apache/incubator-hawq/releases{/id}","created_at":"2015-09-05T07:00:05Z","updated_at":"2015-11-11T15:38:37Z","pushed_at":"2015-11-12T08:33:11Z","git_url":"git://github.com/apache/incubator-hawq.git","ssh_url":"git@github.com:apache/incubator-hawq.git","clone_url":"https://github.com/apache/incubator-hawq.git","svn_url":"https://github.com/apache/incubator-hawq","homepage":null,"size":42991,"stargazers_count":72,"watchers_count":72,"language":"C","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":66,"mirror_url":"git://git.apache.org/incubator-hawq.git","open_issues_count":14,"forks":66,"open_issues":14,"watchers":72,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/incubator-hawq/pulls/92"},"html":{"href":"https://github.com/apache/incubator-hawq/pull/92"},"issue":{"href":"https://api.github.com/repos/apache/incubator-hawq/issues/92"},"comments":{"href":"https://api.github.com/repos/apache/incubator-hawq/issues/92/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/incubator-hawq/pulls/92/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/incubator-hawq/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/incubator-hawq/pulls/92/commits"},"statuses":{"href":"https://api.github.com/repos/apache/incubator-hawq/statuses/49b3e448436c32561af2fc16749ec484216770d6"}}}}