{"action":"created","comment":{"url":"https://api.github.com/repos/druid-io/druid/pulls/comments/44624511","id":44624511,"diff_hunk":"@@ -0,0 +1,205 @@\n+/*\n+* Licensed to Metamarkets Group Inc. (Metamarkets) under one\n+* or more contributor license agreements. See the NOTICE file\n+* distributed with this work for additional information\n+* regarding copyright ownership. Metamarkets licenses this file\n+* to you under the Apache License, Version 2.0 (the\n+* \"License\"); you may not use this file except in compliance\n+* with the License. You may obtain a copy of the License at\n+*\n+* http://www.apache.org/licenses/LICENSE-2.0\n+*\n+* Unless required by applicable law or agreed to in writing,\n+* software distributed under the License is distributed on an\n+* \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n+* KIND, either express or implied. See the License for the\n+* specific language governing permissions and limitations\n+* under the License.\n+*/\n+\n+package io.druid.segment;\n+\n+import com.fasterxml.jackson.annotation.JsonProperty;\n+import com.metamx.common.logger.Logger;\n+import io.druid.query.aggregation.AggregatorFactory;\n+import io.druid.query.aggregation.AggregatorFactoryNotMergeableException;\n+\n+import java.util.ArrayList;\n+import java.util.Arrays;\n+import java.util.HashMap;\n+import java.util.LinkedHashMap;\n+import java.util.List;\n+import java.util.Map;\n+\n+/**\n+ */\n+public class Metadata\n+{\n+\n+  private static final Logger LOG = new Logger(Metadata.class);\n+\n+  // container is used for arbitrary key-value pairs in segment metadata e.g.\n+  // kafka firehose uses it to store commit offset\n+  @JsonProperty\n+  private final Map<String, Object> container;\n+\n+  @JsonProperty\n+  private AggregatorFactory[] aggregators;\n+\n+  public Metadata()\n+  {\n+    container = new HashMap<>();\n+  }\n+\n+  public AggregatorFactory[] getAggregators()\n+  {\n+    return aggregators;\n+  }\n+\n+  public Metadata setAggregators(AggregatorFactory[] aggregators)\n+  {\n+    this.aggregators = aggregators;\n+    return this;\n+  }\n+\n+  public Metadata putAll(Map<String, Object> other)\n+  {\n+    if (other != null) {\n+      container.putAll(other);\n+    }\n+    return this;\n+  }\n+\n+  public Object get(String key)\n+  {\n+    return container.get(key);\n+  }\n+\n+  public Metadata put(String key, Object value)\n+  {\n+    if (value != null) {\n+      container.put(key, value);\n+    }\n+    return this;\n+  }\n+\n+  // arbitrary key-value pairs from the metadata just follow the semantics of last one wins if same\n+  // key exists in multiple input Metadata containers\n+  // for others e.g. Aggregators, appropriate merging is done\n+  public static Metadata merge(\n+      List<Metadata> toBeMerged,\n+      AggregatorFactory[] overrideMergedAggregators\n+  )\n+  {\n+    if (toBeMerged == null || toBeMerged.size() == 0) {\n+      return null;\n+    }\n+\n+    Map<String, Object> mergedContainer = new HashMap<>();\n+    List<AggregatorFactory[]> aggregatorsToMerge = overrideMergedAggregators == null\n+                                                   ? new ArrayList<AggregatorFactory[]>()\n+                                                   : null;\n+\n+    for (Metadata metadata : toBeMerged) {\n+      if (metadata != null) {\n+        if (overrideMergedAggregators == null) {\n+          aggregatorsToMerge.add(metadata.getAggregators());\n+        }\n+        mergedContainer.putAll(metadata.container);\n+      }\n+    }\n+\n+    Metadata result = new Metadata();\n+    if (overrideMergedAggregators == null) {\n+      result.setAggregators(mergeAggregators(aggregatorsToMerge));\n+    } else {\n+      result.setAggregators(overrideMergedAggregators);\n+    }\n+    result.container.putAll(mergedContainer);\n+    return result;\n+\n+  }\n+\n+  /**\n+   * Merges the list of AggregatorFactory[] (presumable from metadata of some segments being merged) and\n+   * returns merged AggregatorFactory[] (for the metadata for merged segment).\n+   * Null is returned if it is not possible to do the merging for any of the following reason.\n+   * - one of the element in input list is null i.e. aggregators for one the segments being merged is unknown\n+   * - AggregatorFactory of same name can not be merged if they are not compatible\n+   *\n+   * @param aggregatorsList\n+   *\n+   * @return merged AggregatorFactory[] or Null if merging is not possible.\n+   */\n+  public static AggregatorFactory[] mergeAggregators(List<AggregatorFactory[]> aggregatorsList)","path":"processing/src/main/java/io/druid/segment/Metadata.java","position":134,"original_position":134,"commit_id":"96336674de1f693a8ff875a182b88451924727c2","original_commit_id":"96336674de1f693a8ff875a182b88451924727c2","user":{"login":"himanshug","id":246308,"avatar_url":"https://avatars.githubusercontent.com/u/246308?v=3","gravatar_id":"","url":"https://api.github.com/users/himanshug","html_url":"https://github.com/himanshug","followers_url":"https://api.github.com/users/himanshug/followers","following_url":"https://api.github.com/users/himanshug/following{/other_user}","gists_url":"https://api.github.com/users/himanshug/gists{/gist_id}","starred_url":"https://api.github.com/users/himanshug/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/himanshug/subscriptions","organizations_url":"https://api.github.com/users/himanshug/orgs","repos_url":"https://api.github.com/users/himanshug/repos","events_url":"https://api.github.com/users/himanshug/events{/privacy}","received_events_url":"https://api.github.com/users/himanshug/received_events","type":"User","site_admin":false},"body":"I am fine with moving it there.","created_at":"2015-11-12T06:15:44Z","updated_at":"2015-11-12T06:15:44Z","html_url":"https://github.com/druid-io/druid/pull/1728#discussion_r44624511","pull_request_url":"https://api.github.com/repos/druid-io/druid/pulls/1728","_links":{"self":{"href":"https://api.github.com/repos/druid-io/druid/pulls/comments/44624511"},"html":{"href":"https://github.com/druid-io/druid/pull/1728#discussion_r44624511"},"pull_request":{"href":"https://api.github.com/repos/druid-io/druid/pulls/1728"}}},"pull_request":{"url":"https://api.github.com/repos/druid-io/druid/pulls/1728","id":44946003,"html_url":"https://github.com/druid-io/druid/pull/1728","diff_url":"https://github.com/druid-io/druid/pull/1728.diff","patch_url":"https://github.com/druid-io/druid/pull/1728.patch","issue_url":"https://api.github.com/repos/druid-io/druid/issues/1728","number":1728,"state":"open","locked":false,"title":"Store AggregatorFactory[] in segment metadata","user":{"login":"himanshug","id":246308,"avatar_url":"https://avatars.githubusercontent.com/u/246308?v=3","gravatar_id":"","url":"https://api.github.com/users/himanshug","html_url":"https://github.com/himanshug","followers_url":"https://api.github.com/users/himanshug/followers","following_url":"https://api.github.com/users/himanshug/following{/other_user}","gists_url":"https://api.github.com/users/himanshug/gists{/gist_id}","starred_url":"https://api.github.com/users/himanshug/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/himanshug/subscriptions","organizations_url":"https://api.github.com/users/himanshug/orgs","repos_url":"https://api.github.com/users/himanshug/repos","events_url":"https://api.github.com/users/himanshug/events{/privacy}","received_events_url":"https://api.github.com/users/himanshug/received_events","type":"User","site_admin":false},"body":"This PR adds the list of aggregators used to create a segment to it's metadata which is persisted in metadata.drd file. This feature aids\r\n1) implementation of hadoop \"reindexing/merging\" task so that it can merge intervals automatically with minimal user input.\r\n2) UI implementations to present a list of aggregators to users\r\n\r\nAlso, this PR improves the segment metadata framework which allows adding more stuff easily  e.g. QueryGranularity (not added in this PR though)\r\n\r\nIn order to implement this feature, following notable changes were made.\r\n1) AggregatorFactory is made an abstract class, a new method was added to AggregatorFactory with a default implementation (also AggregatorFactory was converted to an abstract class), this is required in order to correctly \"merge\"\r\naggregator factories. This is a `backward incompatible` change.\r\n```\r\n/**\r\n * Returns an AggregatorFactory that can be used to merge the output of aggregators from this factory and\r\n * other factory.\r\n * This method is relevant only for AggregatorFactory which can be used at ingestion time.\r\n *\r\n * @return a new Factory that can be used for merging the output of aggregators from this factory and other.\r\n */\r\npublic AggregatorFactory getMergingFactory(AggregatorFactory other) throws AggregatorFactoryNotMergeableException\r\n\r\n\r\n```\r\n\r\n2) User can optionally supply the list of Aggregators to AppendTask so as to explicitly specify list of aggregators to be stored inside the segment metadata of final appended segment.\r\n\r\n","created_at":"2015-09-14T15:43:52Z","updated_at":"2015-11-12T06:15:44Z","closed_at":null,"merged_at":null,"merge_commit_sha":"d894c9f31e98a3ede5082e8af265c6914890b1cf","assignee":null,"milestone":{"url":"https://api.github.com/repos/druid-io/druid/milestones/14","html_url":"https://github.com/druid-io/druid/milestones/0.9.0","labels_url":"https://api.github.com/repos/druid-io/druid/milestones/14/labels","id":1092727,"number":14,"title":"0.9.0","description":"","creator":{"login":"xvrl","id":815147,"avatar_url":"https://avatars.githubusercontent.com/u/815147?v=3","gravatar_id":"","url":"https://api.github.com/users/xvrl","html_url":"https://github.com/xvrl","followers_url":"https://api.github.com/users/xvrl/followers","following_url":"https://api.github.com/users/xvrl/following{/other_user}","gists_url":"https://api.github.com/users/xvrl/gists{/gist_id}","starred_url":"https://api.github.com/users/xvrl/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/xvrl/subscriptions","organizations_url":"https://api.github.com/users/xvrl/orgs","repos_url":"https://api.github.com/users/xvrl/repos","events_url":"https://api.github.com/users/xvrl/events{/privacy}","received_events_url":"https://api.github.com/users/xvrl/received_events","type":"User","site_admin":false},"open_issues":14,"closed_issues":51,"state":"open","created_at":"2015-05-01T16:53:57Z","updated_at":"2015-11-11T20:28:45Z","due_on":null,"closed_at":null},"commits_url":"https://api.github.com/repos/druid-io/druid/pulls/1728/commits","review_comments_url":"https://api.github.com/repos/druid-io/druid/pulls/1728/comments","review_comment_url":"https://api.github.com/repos/druid-io/druid/pulls/comments{/number}","comments_url":"https://api.github.com/repos/druid-io/druid/issues/1728/comments","statuses_url":"https://api.github.com/repos/druid-io/druid/statuses/96336674de1f693a8ff875a182b88451924727c2","head":{"label":"himanshug:aggregators_in_segment_metadata","ref":"aggregators_in_segment_metadata","sha":"96336674de1f693a8ff875a182b88451924727c2","user":{"login":"himanshug","id":246308,"avatar_url":"https://avatars.githubusercontent.com/u/246308?v=3","gravatar_id":"","url":"https://api.github.com/users/himanshug","html_url":"https://github.com/himanshug","followers_url":"https://api.github.com/users/himanshug/followers","following_url":"https://api.github.com/users/himanshug/following{/other_user}","gists_url":"https://api.github.com/users/himanshug/gists{/gist_id}","starred_url":"https://api.github.com/users/himanshug/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/himanshug/subscriptions","organizations_url":"https://api.github.com/users/himanshug/orgs","repos_url":"https://api.github.com/users/himanshug/repos","events_url":"https://api.github.com/users/himanshug/events{/privacy}","received_events_url":"https://api.github.com/users/himanshug/received_events","type":"User","site_admin":false},"repo":{"id":23076411,"name":"druid","full_name":"himanshug/druid","owner":{"login":"himanshug","id":246308,"avatar_url":"https://avatars.githubusercontent.com/u/246308?v=3","gravatar_id":"","url":"https://api.github.com/users/himanshug","html_url":"https://github.com/himanshug","followers_url":"https://api.github.com/users/himanshug/followers","following_url":"https://api.github.com/users/himanshug/following{/other_user}","gists_url":"https://api.github.com/users/himanshug/gists{/gist_id}","starred_url":"https://api.github.com/users/himanshug/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/himanshug/subscriptions","organizations_url":"https://api.github.com/users/himanshug/orgs","repos_url":"https://api.github.com/users/himanshug/repos","events_url":"https://api.github.com/users/himanshug/events{/privacy}","received_events_url":"https://api.github.com/users/himanshug/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/himanshug/druid","description":"Real²time Exploratory Analytics on Large Datasets","fork":true,"url":"https://api.github.com/repos/himanshug/druid","forks_url":"https://api.github.com/repos/himanshug/druid/forks","keys_url":"https://api.github.com/repos/himanshug/druid/keys{/key_id}","collaborators_url":"https://api.github.com/repos/himanshug/druid/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/himanshug/druid/teams","hooks_url":"https://api.github.com/repos/himanshug/druid/hooks","issue_events_url":"https://api.github.com/repos/himanshug/druid/issues/events{/number}","events_url":"https://api.github.com/repos/himanshug/druid/events","assignees_url":"https://api.github.com/repos/himanshug/druid/assignees{/user}","branches_url":"https://api.github.com/repos/himanshug/druid/branches{/branch}","tags_url":"https://api.github.com/repos/himanshug/druid/tags","blobs_url":"https://api.github.com/repos/himanshug/druid/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/himanshug/druid/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/himanshug/druid/git/refs{/sha}","trees_url":"https://api.github.com/repos/himanshug/druid/git/trees{/sha}","statuses_url":"https://api.github.com/repos/himanshug/druid/statuses/{sha}","languages_url":"https://api.github.com/repos/himanshug/druid/languages","stargazers_url":"https://api.github.com/repos/himanshug/druid/stargazers","contributors_url":"https://api.github.com/repos/himanshug/druid/contributors","subscribers_url":"https://api.github.com/repos/himanshug/druid/subscribers","subscription_url":"https://api.github.com/repos/himanshug/druid/subscription","commits_url":"https://api.github.com/repos/himanshug/druid/commits{/sha}","git_commits_url":"https://api.github.com/repos/himanshug/druid/git/commits{/sha}","comments_url":"https://api.github.com/repos/himanshug/druid/comments{/number}","issue_comment_url":"https://api.github.com/repos/himanshug/druid/issues/comments{/number}","contents_url":"https://api.github.com/repos/himanshug/druid/contents/{+path}","compare_url":"https://api.github.com/repos/himanshug/druid/compare/{base}...{head}","merges_url":"https://api.github.com/repos/himanshug/druid/merges","archive_url":"https://api.github.com/repos/himanshug/druid/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/himanshug/druid/downloads","issues_url":"https://api.github.com/repos/himanshug/druid/issues{/number}","pulls_url":"https://api.github.com/repos/himanshug/druid/pulls{/number}","milestones_url":"https://api.github.com/repos/himanshug/druid/milestones{/number}","notifications_url":"https://api.github.com/repos/himanshug/druid/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/himanshug/druid/labels{/name}","releases_url":"https://api.github.com/repos/himanshug/druid/releases{/id}","created_at":"2014-08-18T15:28:53Z","updated_at":"2015-02-23T20:08:38Z","pushed_at":"2015-11-12T06:04:38Z","git_url":"git://github.com/himanshug/druid.git","ssh_url":"git@github.com:himanshug/druid.git","clone_url":"https://github.com/himanshug/druid.git","svn_url":"https://github.com/himanshug/druid","homepage":"http://druid.io","size":98612,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"druid-io:master","ref":"master","sha":"379ca87e6ad888c60da7ed2149f393ac5320a254","user":{"login":"druid-io","id":4203474,"avatar_url":"https://avatars.githubusercontent.com/u/4203474?v=3","gravatar_id":"","url":"https://api.github.com/users/druid-io","html_url":"https://github.com/druid-io","followers_url":"https://api.github.com/users/druid-io/followers","following_url":"https://api.github.com/users/druid-io/following{/other_user}","gists_url":"https://api.github.com/users/druid-io/gists{/gist_id}","starred_url":"https://api.github.com/users/druid-io/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/druid-io/subscriptions","organizations_url":"https://api.github.com/users/druid-io/orgs","repos_url":"https://api.github.com/users/druid-io/repos","events_url":"https://api.github.com/users/druid-io/events{/privacy}","received_events_url":"https://api.github.com/users/druid-io/received_events","type":"Organization","site_admin":false},"repo":{"id":6358188,"name":"druid","full_name":"druid-io/druid","owner":{"login":"druid-io","id":4203474,"avatar_url":"https://avatars.githubusercontent.com/u/4203474?v=3","gravatar_id":"","url":"https://api.github.com/users/druid-io","html_url":"https://github.com/druid-io","followers_url":"https://api.github.com/users/druid-io/followers","following_url":"https://api.github.com/users/druid-io/following{/other_user}","gists_url":"https://api.github.com/users/druid-io/gists{/gist_id}","starred_url":"https://api.github.com/users/druid-io/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/druid-io/subscriptions","organizations_url":"https://api.github.com/users/druid-io/orgs","repos_url":"https://api.github.com/users/druid-io/repos","events_url":"https://api.github.com/users/druid-io/events{/privacy}","received_events_url":"https://api.github.com/users/druid-io/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/druid-io/druid","description":"Column oriented distributed data store ideal for powering interactive applications","fork":false,"url":"https://api.github.com/repos/druid-io/druid","forks_url":"https://api.github.com/repos/druid-io/druid/forks","keys_url":"https://api.github.com/repos/druid-io/druid/keys{/key_id}","collaborators_url":"https://api.github.com/repos/druid-io/druid/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/druid-io/druid/teams","hooks_url":"https://api.github.com/repos/druid-io/druid/hooks","issue_events_url":"https://api.github.com/repos/druid-io/druid/issues/events{/number}","events_url":"https://api.github.com/repos/druid-io/druid/events","assignees_url":"https://api.github.com/repos/druid-io/druid/assignees{/user}","branches_url":"https://api.github.com/repos/druid-io/druid/branches{/branch}","tags_url":"https://api.github.com/repos/druid-io/druid/tags","blobs_url":"https://api.github.com/repos/druid-io/druid/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/druid-io/druid/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/druid-io/druid/git/refs{/sha}","trees_url":"https://api.github.com/repos/druid-io/druid/git/trees{/sha}","statuses_url":"https://api.github.com/repos/druid-io/druid/statuses/{sha}","languages_url":"https://api.github.com/repos/druid-io/druid/languages","stargazers_url":"https://api.github.com/repos/druid-io/druid/stargazers","contributors_url":"https://api.github.com/repos/druid-io/druid/contributors","subscribers_url":"https://api.github.com/repos/druid-io/druid/subscribers","subscription_url":"https://api.github.com/repos/druid-io/druid/subscription","commits_url":"https://api.github.com/repos/druid-io/druid/commits{/sha}","git_commits_url":"https://api.github.com/repos/druid-io/druid/git/commits{/sha}","comments_url":"https://api.github.com/repos/druid-io/druid/comments{/number}","issue_comment_url":"https://api.github.com/repos/druid-io/druid/issues/comments{/number}","contents_url":"https://api.github.com/repos/druid-io/druid/contents/{+path}","compare_url":"https://api.github.com/repos/druid-io/druid/compare/{base}...{head}","merges_url":"https://api.github.com/repos/druid-io/druid/merges","archive_url":"https://api.github.com/repos/druid-io/druid/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/druid-io/druid/downloads","issues_url":"https://api.github.com/repos/druid-io/druid/issues{/number}","pulls_url":"https://api.github.com/repos/druid-io/druid/pulls{/number}","milestones_url":"https://api.github.com/repos/druid-io/druid/milestones{/number}","notifications_url":"https://api.github.com/repos/druid-io/druid/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/druid-io/druid/labels{/name}","releases_url":"https://api.github.com/repos/druid-io/druid/releases{/id}","created_at":"2012-10-23T19:08:07Z","updated_at":"2015-11-12T04:46:44Z","pushed_at":"2015-11-12T06:04:38Z","git_url":"git://github.com/druid-io/druid.git","ssh_url":"git@github.com:druid-io/druid.git","clone_url":"https://github.com/druid-io/druid.git","svn_url":"https://github.com/druid-io/druid","homepage":"http://druid.io","size":184200,"stargazers_count":2155,"watchers_count":2155,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":492,"mirror_url":null,"open_issues_count":224,"forks":492,"open_issues":224,"watchers":2155,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/druid-io/druid/pulls/1728"},"html":{"href":"https://github.com/druid-io/druid/pull/1728"},"issue":{"href":"https://api.github.com/repos/druid-io/druid/issues/1728"},"comments":{"href":"https://api.github.com/repos/druid-io/druid/issues/1728/comments"},"review_comments":{"href":"https://api.github.com/repos/druid-io/druid/pulls/1728/comments"},"review_comment":{"href":"https://api.github.com/repos/druid-io/druid/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/druid-io/druid/pulls/1728/commits"},"statuses":{"href":"https://api.github.com/repos/druid-io/druid/statuses/96336674de1f693a8ff875a182b88451924727c2"}}}}