{"action":"created","comment":{"url":"https://api.github.com/repos/caskdata/cdap/pulls/comments/24434884","id":24434884,"diff_hunk":"@@ -0,0 +1,160 @@\n+/*\n+ * Copyright 2015 Cask Data, Inc.\n+ *\n+ * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n+ * use this file except in compliance with the License. You may obtain a copy of\n+ * the License at\n+ *\n+ * http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n+ * License for the specific language governing permissions and limitations under\n+ * the License.\n+ */\n+\n+package co.cask.cdap.metrics.store;\n+\n+import co.cask.cdap.common.conf.Constants;\n+import co.cask.cdap.metrics.store.cube.Aggregation;\n+import co.cask.cdap.metrics.store.cube.Cube;\n+import co.cask.cdap.metrics.store.cube.CubeFact;\n+import co.cask.cdap.metrics.store.cube.CubeQuery;\n+import co.cask.cdap.metrics.store.cube.DefaultAggregation;\n+import co.cask.cdap.metrics.store.cube.DefaultCube;\n+import co.cask.cdap.metrics.store.cube.FactTableSupplier;\n+import co.cask.cdap.metrics.store.cube.TimeSeries;\n+import co.cask.cdap.metrics.store.timeseries.FactTable;\n+import co.cask.cdap.metrics.store.timeseries.MeasureType;\n+import co.cask.cdap.metrics.store.timeseries.TimeValue;\n+import co.cask.cdap.metrics.transport.MetricType;\n+import co.cask.cdap.metrics.transport.MetricValue;\n+import com.google.common.base.Supplier;\n+import com.google.common.base.Suppliers;\n+import com.google.common.collect.ImmutableList;\n+import com.google.common.collect.ImmutableMap;\n+import com.google.common.collect.Lists;\n+import com.google.common.collect.Maps;\n+import com.google.inject.Inject;\n+\n+import java.util.Collection;\n+import java.util.List;\n+import java.util.Map;\n+\n+/**\n+ *\n+ */\n+public class DefaultMetricStore implements MetricStore {\n+  private static final String PROGRAM_LEVEL2 = \"pr2\";\n+  private static final String PROGRAM_LEVEL3 = \"pr3\";\n+  private static final String PROGRAM_LEVEL4 = \"pr4\";\n+\n+  private final Supplier<Cube> cube;\n+  private final Map<String, String> tagMapping;\n+\n+  @Inject\n+  public DefaultMetricStore(final MetricDatasetFactory dsFactory) {\n+    final FactTableSupplier factTableSupplier = new FactTableSupplier() {\n+      @Override\n+      public FactTable get(int resolution, int ignoredRollTime) {\n+        // roll time will be taken from configuration todo: clean this up\n+        return dsFactory.get(resolution);\n+      }\n+    };\n+    this.cube = Suppliers.memoize(new Supplier<Cube>() {\n+      @Override\n+      public Cube get() {\n+        // 1 sec, 1 min, 1 hour and \"all time totals\"\n+        return new DefaultCube(new int[] {1, 60, 3600, Integer.MAX_VALUE}, factTableSupplier, createAggregations());\n+      }\n+    });\n+\n+    this.tagMapping = ImmutableMap.of(\n+      // flow\n+      Constants.Metrics.Tag.FLOWLET, PROGRAM_LEVEL2,\n+      Constants.Metrics.Tag.FLOWLET_QUEUE, PROGRAM_LEVEL3,\n+      // mapreduce\n+      Constants.Metrics.Tag.MR_TASK_TYPE, PROGRAM_LEVEL2,\n+      // service\n+      Constants.Metrics.Tag.SERVICE_RUNNABLE, PROGRAM_LEVEL2\n+    );\n+  }\n+\n+  private static List<Aggregation> createAggregations() {\n+    List<Aggregation> aggs = Lists.newLinkedList();\n+\n+    // <cluster metrics>, e.g. storage used\n+    aggs.add(new DefaultAggregation(ImmutableList.of(\n+      Constants.Metrics.Tag.NAMESPACE, Constants.Metrics.Tag.CLUSTER_METRICS)));\n+\n+    // app, prg type, prg name, ...\n+    aggs.add(new DefaultAggregation(ImmutableList.of(\n+      Constants.Metrics.Tag.NAMESPACE, Constants.Metrics.Tag.APP,\n+      // todo: do we even need program type? seems like program name unique within app across program types\n+      Constants.Metrics.Tag.PROGRAM_TYPE, Constants.Metrics.Tag.PROGRAM, Constants.Metrics.Tag.RUN_ID,\n+      PROGRAM_LEVEL2, PROGRAM_LEVEL3, PROGRAM_LEVEL4, Constants.Metrics.Tag.DATASET)));\n+\n+    // component, handler, method\n+    aggs.add(new DefaultAggregation(ImmutableList.of(\n+      Constants.Metrics.Tag.NAMESPACE,\n+      Constants.Metrics.Tag.COMPONENT, Constants.Metrics.Tag.HANDLER, Constants.Metrics.Tag.METHOD)));\n+\n+    // component, handler, method, stream (for stream only) todo: seems like emitted context is wrong, review...\n+    aggs.add(new DefaultAggregation(ImmutableList.of(\n+      Constants.Metrics.Tag.NAMESPACE,\n+      Constants.Metrics.Tag.COMPONENT, Constants.Metrics.Tag.HANDLER, Constants.Metrics.Tag.METHOD,\n+      Constants.Metrics.Tag.STREAM),\n+                                    // i.e. for stream only\n+                                    ImmutableList.of(Constants.Metrics.Tag.STREAM)));\n+\n+    // dataset\n+    aggs.add(new DefaultAggregation(ImmutableList.of(Constants.Metrics.Tag.NAMESPACE, Constants.Metrics.Tag.DATASET)));\n+\n+    return aggs;\n+  }\n+\n+  @Override\n+  public void add(MetricValue metricValue) throws Exception {\n+    String scope = metricValue.getTags().get(Constants.Metrics.Tag.SCOPE);\n+    String measureName = (scope == null ? \"system.\" : scope + \".\") + metricValue.getName();\n+\n+    CubeFact fact = new CubeFact(replaceTagsIfNeeded(metricValue.getTags()),\n+                                 toMeasureType(metricValue.getType()), measureName,\n+                                 new TimeValue(metricValue.getTimestamp(), metricValue.getValue()));\n+    cube.get().add(fact);\n+  }\n+\n+  private Map<String, String> replaceTagsIfNeeded(Map<String, String> original) {\n+    // replace tags using mapping if needed todo: hacky and not efficient?\n+    Map<String, String> tags = Maps.newHashMap();\n+    for (Map.Entry<String, String> tagValue : original.entrySet()) {\n+      String tagNameReplacement = tagMapping.get(tagValue.getKey());\n+      tags.put(tagNameReplacement == null ? tagValue.getKey() : tagNameReplacement, tagValue.getValue());\n+    }\n+    return tags;\n+  }\n+\n+  @Override\n+  public Collection<TimeSeries> query(CubeQuery query) throws Exception {\n+    CubeQuery q = new CubeQuery(query, replaceTagsIfNeeded(query.getSliceByTags()));\n+    return cube.get().query(q);\n+  }\n+\n+  @Override\n+  public void deleteBefore(long timestamp) {\n+    // todo: implement metric ttl\n+  }\n+\n+  private MeasureType toMeasureType(MetricType type) {\n+    switch (type) {\n+      case COUNTER:\n+        return MeasureType.COUNTER;\n+      case GAUGE:\n+        return MeasureType.GAUGE;\n+      default:\n+        // should never ha","path":"cdap-watchdog/src/main/java/co/cask/cdap/metrics/store/DefaultMetricStore.java","position":156,"original_position":156,"commit_id":"145a649642f29e0cabbf08f1f7e52a40aa6dbaf1","original_commit_id":"145a649642f29e0cabbf08f1f7e52a40aa6dbaf1","user":{"login":"gsps1","id":1328150,"avatar_url":"https://avatars.githubusercontent.com/u/1328150?v=3","gravatar_id":"","url":"https://api.github.com/users/gsps1","html_url":"https://github.com/gsps1","followers_url":"https://api.github.com/users/gsps1/followers","following_url":"https://api.github.com/users/gsps1/following{/other_user}","gists_url":"https://api.github.com/users/gsps1/gists{/gist_id}","starred_url":"https://api.github.com/users/gsps1/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gsps1/subscriptions","organizations_url":"https://api.github.com/users/gsps1/orgs","repos_url":"https://api.github.com/users/gsps1/repos","events_url":"https://api.github.com/users/gsps1/events{/privacy}","received_events_url":"https://api.github.com/users/gsps1/received_events","type":"User","site_admin":false},"body":"happen","created_at":"2015-02-10T18:10:29Z","updated_at":"2015-02-10T18:10:29Z","html_url":"https://github.com/caskdata/cdap/pull/1251#discussion_r24434884","pull_request_url":"https://api.github.com/repos/caskdata/cdap/pulls/1251","_links":{"self":{"href":"https://api.github.com/repos/caskdata/cdap/pulls/comments/24434884"},"html":{"href":"https://github.com/caskdata/cdap/pull/1251#discussion_r24434884"},"pull_request":{"href":"https://api.github.com/repos/caskdata/cdap/pulls/1251"}}},"pull_request":{"url":"https://api.github.com/repos/caskdata/cdap/pulls/1251","id":28856636,"html_url":"https://github.com/caskdata/cdap/pull/1251","diff_url":"https://github.com/caskdata/cdap/pull/1251.diff","patch_url":"https://github.com/caskdata/cdap/pull/1251.patch","issue_url":"https://api.github.com/repos/caskdata/cdap/issues/1251","number":1251,"state":"open","locked":false,"title":"metric store initial integration, work in progress","user":{"login":"abaranau","id":434133,"avatar_url":"https://avatars.githubusercontent.com/u/434133?v=3","gravatar_id":"","url":"https://api.github.com/users/abaranau","html_url":"https://github.com/abaranau","followers_url":"https://api.github.com/users/abaranau/followers","following_url":"https://api.github.com/users/abaranau/following{/other_user}","gists_url":"https://api.github.com/users/abaranau/gists{/gist_id}","starred_url":"https://api.github.com/users/abaranau/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/abaranau/subscriptions","organizations_url":"https://api.github.com/users/abaranau/orgs","repos_url":"https://api.github.com/users/abaranau/repos","events_url":"https://api.github.com/users/abaranau/events{/privacy}","received_events_url":"https://api.github.com/users/abaranau/received_events","type":"User","site_admin":false},"body":"Here's the branching strategy for metric store change. \r\n* metric-strore-integrated is an integration branch for the feature\r\n* we'll create PRs and review them against this branch\r\n* so, all code in it is reviewed\r\n* when all done - we'll merge it into develop\r\n\r\nIn this PR: at least these are missing (hence there are todos in code):\r\n\r\n* upgrade metric datasets\r\n* ttl for non-hbase metric datasets\r\n* interpolators\r\n* delete metrics\r\n* metrics discovery\r\n* unit-test framework\r\n* metrics.data package - move\r\n* queue length - overall all unit-tests to pass\r\n* support limit in cube/fact table\r\n* remove + .v2 in dataset names\r\n","created_at":"2015-02-07T01:58:45Z","updated_at":"2015-02-10T18:10:29Z","closed_at":null,"merged_at":null,"merge_commit_sha":"2f92e077f27b0df8e330318b89f7b73e097c746f","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/caskdata/cdap/pulls/1251/commits","review_comments_url":"https://api.github.com/repos/caskdata/cdap/pulls/1251/comments","review_comment_url":"https://api.github.com/repos/caskdata/cdap/pulls/comments/{number}","comments_url":"https://api.github.com/repos/caskdata/cdap/issues/1251/comments","statuses_url":"https://api.github.com/repos/caskdata/cdap/statuses/145a649642f29e0cabbf08f1f7e52a40aa6dbaf1","head":{"label":"caskdata:feature/metricstore","ref":"feature/metricstore","sha":"145a649642f29e0cabbf08f1f7e52a40aa6dbaf1","user":{"login":"caskdata","id":8123023,"avatar_url":"https://avatars.githubusercontent.com/u/8123023?v=3","gravatar_id":"","url":"https://api.github.com/users/caskdata","html_url":"https://github.com/caskdata","followers_url":"https://api.github.com/users/caskdata/followers","following_url":"https://api.github.com/users/caskdata/following{/other_user}","gists_url":"https://api.github.com/users/caskdata/gists{/gist_id}","starred_url":"https://api.github.com/users/caskdata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/caskdata/subscriptions","organizations_url":"https://api.github.com/users/caskdata/orgs","repos_url":"https://api.github.com/users/caskdata/repos","events_url":"https://api.github.com/users/caskdata/events{/privacy}","received_events_url":"https://api.github.com/users/caskdata/received_events","type":"Organization","site_admin":false},"repo":{"id":22542759,"name":"cdap","full_name":"caskdata/cdap","owner":{"login":"caskdata","id":8123023,"avatar_url":"https://avatars.githubusercontent.com/u/8123023?v=3","gravatar_id":"","url":"https://api.github.com/users/caskdata","html_url":"https://github.com/caskdata","followers_url":"https://api.github.com/users/caskdata/followers","following_url":"https://api.github.com/users/caskdata/following{/other_user}","gists_url":"https://api.github.com/users/caskdata/gists{/gist_id}","starred_url":"https://api.github.com/users/caskdata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/caskdata/subscriptions","organizations_url":"https://api.github.com/users/caskdata/orgs","repos_url":"https://api.github.com/users/caskdata/repos","events_url":"https://api.github.com/users/caskdata/events{/privacy}","received_events_url":"https://api.github.com/users/caskdata/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/caskdata/cdap","description":"Cask Data Application Platform (CDAP)","fork":false,"url":"https://api.github.com/repos/caskdata/cdap","forks_url":"https://api.github.com/repos/caskdata/cdap/forks","keys_url":"https://api.github.com/repos/caskdata/cdap/keys{/key_id}","collaborators_url":"https://api.github.com/repos/caskdata/cdap/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/caskdata/cdap/teams","hooks_url":"https://api.github.com/repos/caskdata/cdap/hooks","issue_events_url":"https://api.github.com/repos/caskdata/cdap/issues/events{/number}","events_url":"https://api.github.com/repos/caskdata/cdap/events","assignees_url":"https://api.github.com/repos/caskdata/cdap/assignees{/user}","branches_url":"https://api.github.com/repos/caskdata/cdap/branches{/branch}","tags_url":"https://api.github.com/repos/caskdata/cdap/tags","blobs_url":"https://api.github.com/repos/caskdata/cdap/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/caskdata/cdap/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/caskdata/cdap/git/refs{/sha}","trees_url":"https://api.github.com/repos/caskdata/cdap/git/trees{/sha}","statuses_url":"https://api.github.com/repos/caskdata/cdap/statuses/{sha}","languages_url":"https://api.github.com/repos/caskdata/cdap/languages","stargazers_url":"https://api.github.com/repos/caskdata/cdap/stargazers","contributors_url":"https://api.github.com/repos/caskdata/cdap/contributors","subscribers_url":"https://api.github.com/repos/caskdata/cdap/subscribers","subscription_url":"https://api.github.com/repos/caskdata/cdap/subscription","commits_url":"https://api.github.com/repos/caskdata/cdap/commits{/sha}","git_commits_url":"https://api.github.com/repos/caskdata/cdap/git/commits{/sha}","comments_url":"https://api.github.com/repos/caskdata/cdap/comments{/number}","issue_comment_url":"https://api.github.com/repos/caskdata/cdap/issues/comments/{number}","contents_url":"https://api.github.com/repos/caskdata/cdap/contents/{+path}","compare_url":"https://api.github.com/repos/caskdata/cdap/compare/{base}...{head}","merges_url":"https://api.github.com/repos/caskdata/cdap/merges","archive_url":"https://api.github.com/repos/caskdata/cdap/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/caskdata/cdap/downloads","issues_url":"https://api.github.com/repos/caskdata/cdap/issues{/number}","pulls_url":"https://api.github.com/repos/caskdata/cdap/pulls{/number}","milestones_url":"https://api.github.com/repos/caskdata/cdap/milestones{/number}","notifications_url":"https://api.github.com/repos/caskdata/cdap/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/caskdata/cdap/labels{/name}","releases_url":"https://api.github.com/repos/caskdata/cdap/releases{/id}","created_at":"2014-08-02T09:00:10Z","updated_at":"2015-02-10T00:02:31Z","pushed_at":"2015-02-10T11:06:17Z","git_url":"git://github.com/caskdata/cdap.git","ssh_url":"git@github.com:caskdata/cdap.git","clone_url":"https://github.com/caskdata/cdap.git","svn_url":"https://github.com/caskdata/cdap","homepage":"http://cdap.io","size":774274,"stargazers_count":24,"watchers_count":24,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":true,"forks_count":8,"mirror_url":null,"open_issues_count":13,"forks":8,"open_issues":13,"watchers":24,"default_branch":"develop"}},"base":{"label":"caskdata:feature/metric-store-integrated","ref":"feature/metric-store-integrated","sha":"14a72fb9e1427b6c769cfe3fb699eece8934ecfb","user":{"login":"caskdata","id":8123023,"avatar_url":"https://avatars.githubusercontent.com/u/8123023?v=3","gravatar_id":"","url":"https://api.github.com/users/caskdata","html_url":"https://github.com/caskdata","followers_url":"https://api.github.com/users/caskdata/followers","following_url":"https://api.github.com/users/caskdata/following{/other_user}","gists_url":"https://api.github.com/users/caskdata/gists{/gist_id}","starred_url":"https://api.github.com/users/caskdata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/caskdata/subscriptions","organizations_url":"https://api.github.com/users/caskdata/orgs","repos_url":"https://api.github.com/users/caskdata/repos","events_url":"https://api.github.com/users/caskdata/events{/privacy}","received_events_url":"https://api.github.com/users/caskdata/received_events","type":"Organization","site_admin":false},"repo":{"id":22542759,"name":"cdap","full_name":"caskdata/cdap","owner":{"login":"caskdata","id":8123023,"avatar_url":"https://avatars.githubusercontent.com/u/8123023?v=3","gravatar_id":"","url":"https://api.github.com/users/caskdata","html_url":"https://github.com/caskdata","followers_url":"https://api.github.com/users/caskdata/followers","following_url":"https://api.github.com/users/caskdata/following{/other_user}","gists_url":"https://api.github.com/users/caskdata/gists{/gist_id}","starred_url":"https://api.github.com/users/caskdata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/caskdata/subscriptions","organizations_url":"https://api.github.com/users/caskdata/orgs","repos_url":"https://api.github.com/users/caskdata/repos","events_url":"https://api.github.com/users/caskdata/events{/privacy}","received_events_url":"https://api.github.com/users/caskdata/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/caskdata/cdap","description":"Cask Data Application Platform (CDAP)","fork":false,"url":"https://api.github.com/repos/caskdata/cdap","forks_url":"https://api.github.com/repos/caskdata/cdap/forks","keys_url":"https://api.github.com/repos/caskdata/cdap/keys{/key_id}","collaborators_url":"https://api.github.com/repos/caskdata/cdap/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/caskdata/cdap/teams","hooks_url":"https://api.github.com/repos/caskdata/cdap/hooks","issue_events_url":"https://api.github.com/repos/caskdata/cdap/issues/events{/number}","events_url":"https://api.github.com/repos/caskdata/cdap/events","assignees_url":"https://api.github.com/repos/caskdata/cdap/assignees{/user}","branches_url":"https://api.github.com/repos/caskdata/cdap/branches{/branch}","tags_url":"https://api.github.com/repos/caskdata/cdap/tags","blobs_url":"https://api.github.com/repos/caskdata/cdap/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/caskdata/cdap/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/caskdata/cdap/git/refs{/sha}","trees_url":"https://api.github.com/repos/caskdata/cdap/git/trees{/sha}","statuses_url":"https://api.github.com/repos/caskdata/cdap/statuses/{sha}","languages_url":"https://api.github.com/repos/caskdata/cdap/languages","stargazers_url":"https://api.github.com/repos/caskdata/cdap/stargazers","contributors_url":"https://api.github.com/repos/caskdata/cdap/contributors","subscribers_url":"https://api.github.com/repos/caskdata/cdap/subscribers","subscription_url":"https://api.github.com/repos/caskdata/cdap/subscription","commits_url":"https://api.github.com/repos/caskdata/cdap/commits{/sha}","git_commits_url":"https://api.github.com/repos/caskdata/cdap/git/commits{/sha}","comments_url":"https://api.github.com/repos/caskdata/cdap/comments{/number}","issue_comment_url":"https://api.github.com/repos/caskdata/cdap/issues/comments/{number}","contents_url":"https://api.github.com/repos/caskdata/cdap/contents/{+path}","compare_url":"https://api.github.com/repos/caskdata/cdap/compare/{base}...{head}","merges_url":"https://api.github.com/repos/caskdata/cdap/merges","archive_url":"https://api.github.com/repos/caskdata/cdap/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/caskdata/cdap/downloads","issues_url":"https://api.github.com/repos/caskdata/cdap/issues{/number}","pulls_url":"https://api.github.com/repos/caskdata/cdap/pulls{/number}","milestones_url":"https://api.github.com/repos/caskdata/cdap/milestones{/number}","notifications_url":"https://api.github.com/repos/caskdata/cdap/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/caskdata/cdap/labels{/name}","releases_url":"https://api.github.com/repos/caskdata/cdap/releases{/id}","created_at":"2014-08-02T09:00:10Z","updated_at":"2015-02-10T00:02:31Z","pushed_at":"2015-02-10T11:06:17Z","git_url":"git://github.com/caskdata/cdap.git","ssh_url":"git@github.com:caskdata/cdap.git","clone_url":"https://github.com/caskdata/cdap.git","svn_url":"https://github.com/caskdata/cdap","homepage":"http://cdap.io","size":774274,"stargazers_count":24,"watchers_count":24,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":true,"forks_count":8,"mirror_url":null,"open_issues_count":13,"forks":8,"open_issues":13,"watchers":24,"default_branch":"develop"}},"_links":{"self":{"href":"https://api.github.com/repos/caskdata/cdap/pulls/1251"},"html":{"href":"https://github.com/caskdata/cdap/pull/1251"},"issue":{"href":"https://api.github.com/repos/caskdata/cdap/issues/1251"},"comments":{"href":"https://api.github.com/repos/caskdata/cdap/issues/1251/comments"},"review_comments":{"href":"https://api.github.com/repos/caskdata/cdap/pulls/1251/comments"},"review_comment":{"href":"https://api.github.com/repos/caskdata/cdap/pulls/comments/{number}"},"commits":{"href":"https://api.github.com/repos/caskdata/cdap/pulls/1251/commits"},"statuses":{"href":"https://api.github.com/repos/caskdata/cdap/statuses/145a649642f29e0cabbf08f1f7e52a40aa6dbaf1"}}}}