{"action":"created","comment":{"url":"https://api.github.com/repos/rackerlabs/blueflood/pulls/comments/29978337","id":29978337,"diff_hunk":"@@ -87,105 +217,207 @@ def fetch(self, start_time, end_time):\n     if not self.metric:\n       return ((start_time, end_time, 1), [])\n     else:\n-      client = Client(self.bf_query_endpoint, self.tenant)\n-      values = client.get_values(self.metric, start_time, end_time)\n-      # value keys in order of preference.\n-      value_res_order = ['average', 'latest', 'numPoints']\n-\n-      # determine the step\n-      minTime = 0x7fffffffffffffff\n-      maxTime = 0\n-      lastTime = 0\n-      step = 1\n-      value_arr = []\n-      for obj in values:\n-        timestamp = obj['timestamp'] / 1000\n-        step = timestamp - lastTime\n-        lastTime = timestamp\n-        minTime = min(minTime, timestamp)\n-        maxTime = max(maxTime, timestamp)\n-        present_keys = [_ for _ in value_res_order if _ in obj]\n-        if present_keys:\n-            value_arr.append(obj[present_keys[0]])\n-\n-      time_info = (minTime, maxTime, step)\n-      return (time_info, value_arr)\n-\n-SECONDS_IN_5MIN = 300\n-SECONDS_IN_20MIN = 1200\n-SECONDS_IN_60MIN = 3600\n-SECONDS_IN_240MIN = 14400\n-SECONDS_IN_1440MIN = 86400\n-\n-class Client(object):\n-  def __init__(self, host, tenant):\n+      client = BluefloodClient(self.bf_query_endpoint, self.tenant,\n+                                  self.enable_submetrics, self.submetric_aliases)\n+      reader = TenantBluefloodReader(self.metric, self.tenant, self.bf_query_endpoint, \n+                                     self.enable_submetrics, self.submetric_aliases)\n+      node = TenantBluefloodLeafNode(self.metric, reader)\n+\n+      time_info, dictionary = client.fetch_multi([node], start_time, end_time)\n+      return (time_info, dictionary[self.metric])\n+\n+class BluefloodClient(object):\n+  def __init__(self, host, tenant, enable_submetrics, submetric_aliases):\n     self.host = host\n     self.tenant = tenant\n+    self.enable_submetrics = enable_submetrics\n+    self.submetric_aliases = submetric_aliases\n+    # This is the maximum number of json characters permitted by the\n+    #  BF multiplot handler.\n+    # The actual max is 8000, but I want to leave a margin for error\n+    #  because I can't be certain of the exact way the json is generated\n+    self.maxlen_per_req = 7000\n+    # this is for: \"'' ,\" that surround each metric\n+    self.overhead_per_metric = 4\n+    self.maxmetrics_per_req = 100\n \n-  def make_request(self, url, payload, headers):\n+  def gen_data_key(self, values):\n+    # Determines which key to use for the data\n+    if not len(values):\n+      return None\n+    value_res_order = ['average', 'latest', 'numPoints']\n+    present_keys = [v for v in value_res_order if v in values[0]]\n+    if present_keys:\n+      return present_keys[0]\n+    else:\n+      return None\n+\n+  def current_datapoint_passed(self, v_iter, ts):\n+    # Determines if the current datapoint is too old to be considered\n+    if not len(v_iter):\n+      return False\n+    datapoint_ts = v_iter[0]['timestamp']/1000\n+    if (ts > datapoint_ts):\n+      return True\n+    return False\n+\n+  def current_datapoint_valid(self, v_iter, data_key, ts, step):\n+    # Determines if the current datapoint be included in the current step\n+    #assumes current_datapoint_passed() is true\n+    if (not len(v_iter)) or not (data_key in v_iter[0]):\n+      return False\n+    datapoint_ts = v_iter[0]['timestamp']/1000\n+    if (datapoint_ts < (ts + step)):\n+      return True\n+    return False\n+\n+  def process_path(self, values, start_time, end_time, step, data_key):\n+    # Generates datapoints in graphite-api format\n+    # Graphite-api requires the points be \"step\" seconds apart in time\n+    #  with Null's interleaved if needed\n+    if not data_key:\n+      print \"No valid keys present\"\n+      return []\n+    v_iter = values\n+    ret_arr = []\n+    for ts in range(start_time, end_time, step):\n+      # Skip datapoints that have already passed\n+      #  NOTE/TODO: this while loop has the effect of dropping all but the first datapoint in each step\n+      #  (Graphite requires exactly one datapoint/step.)  It would be better to rollup the datapoints\n+      #  if there are more than one/step.  However that will only happen when we have full res metrics\n+      #  with sub-second frequencies.  We believe that case is too rare to worry about right now.\n+      while self.current_datapoint_passed(v_iter, ts):\n+        v_iter = v_iter[1:]\n+      if self.current_datapoint_valid(v_iter, data_key, ts, step):\n+        ret_arr.append(v_iter[0][data_key])\n+      else:\n+        ret_arr.append(None)\n+\n+    return ret_arr\n+\n+  def get_multi_endpoint(self, endpoint, tenant):\n+    return \"%s/v2.0/%s/views\" % (endpoint, tenant)\n+\n+  def get_metric_list(self, endpoint, tenant, metric_list, payload, headers):","path":"contrib/graphite/blueflood.py","position":345,"original_position":345,"commit_id":"c21dd403edc118ffe614324f143c433185acd5d3","original_commit_id":"c21dd403edc118ffe614324f143c433185acd5d3","user":{"login":"chinmay-gupte","id":4722931,"avatar_url":"https://avatars.githubusercontent.com/u/4722931?v=3","gravatar_id":"","url":"https://api.github.com/users/chinmay-gupte","html_url":"https://github.com/chinmay-gupte","followers_url":"https://api.github.com/users/chinmay-gupte/followers","following_url":"https://api.github.com/users/chinmay-gupte/following{/other_user}","gists_url":"https://api.github.com/users/chinmay-gupte/gists{/gist_id}","starred_url":"https://api.github.com/users/chinmay-gupte/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chinmay-gupte/subscriptions","organizations_url":"https://api.github.com/users/chinmay-gupte/orgs","repos_url":"https://api.github.com/users/chinmay-gupte/repos","events_url":"https://api.github.com/users/chinmay-gupte/events{/privacy}","received_events_url":"https://api.github.com/users/chinmay-gupte/received_events","type":"User","site_admin":false},"body":"This name of the function might be a bit confusing, as it prompts to metrics_list api call. Can we have a better name of it, may be `get_metric_data` or something like that?","created_at":"2015-05-08T21:28:02Z","updated_at":"2015-05-08T21:28:02Z","html_url":"https://github.com/rackerlabs/blueflood/pull/460#discussion_r29978337","pull_request_url":"https://api.github.com/repos/rackerlabs/blueflood/pulls/460","_links":{"self":{"href":"https://api.github.com/repos/rackerlabs/blueflood/pulls/comments/29978337"},"html":{"href":"https://github.com/rackerlabs/blueflood/pull/460#discussion_r29978337"},"pull_request":{"href":"https://api.github.com/repos/rackerlabs/blueflood/pulls/460"}}},"pull_request":{"url":"https://api.github.com/repos/rackerlabs/blueflood/pulls/460","id":34505636,"html_url":"https://github.com/rackerlabs/blueflood/pull/460","diff_url":"https://github.com/rackerlabs/blueflood/pull/460.diff","patch_url":"https://github.com/rackerlabs/blueflood/pull/460.patch","issue_url":"https://api.github.com/repos/rackerlabs/blueflood/issues/460","number":460,"state":"open","locked":false,"title":"Multiplot/fetch_multi changes","user":{"login":"GeorgeJahad","id":86830,"avatar_url":"https://avatars.githubusercontent.com/u/86830?v=3","gravatar_id":"","url":"https://api.github.com/users/GeorgeJahad","html_url":"https://github.com/GeorgeJahad","followers_url":"https://api.github.com/users/GeorgeJahad/followers","following_url":"https://api.github.com/users/GeorgeJahad/following{/other_user}","gists_url":"https://api.github.com/users/GeorgeJahad/gists{/gist_id}","starred_url":"https://api.github.com/users/GeorgeJahad/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/GeorgeJahad/subscriptions","organizations_url":"https://api.github.com/users/GeorgeJahad/orgs","repos_url":"https://api.github.com/users/GeorgeJahad/repos","events_url":"https://api.github.com/users/GeorgeJahad/events{/privacy}","received_events_url":"https://api.github.com/users/GeorgeJahad/received_events","type":"User","site_admin":false},"body":"Made a number of fixes/enhancements to the finder:\r\n\r\n1. Graphite requires the datapoints returned by the finder to be \r\nequidistant in time, (interleaved with Null datapoints if necessary.)\r\nWe weren't doing this so the timescale was off on graphs with sparse data.\r\n\r\n2. Improved performance significantly by switching to fetch_multi/multiplot.\r\n(Note this required updating the graphite-api code from version 1.0.1 to \r\nHead of master.  Since we also had to patch that code I created a new repo\r\nfor it here: https://github.com/rackerlabs/graphite-api/tree/george/fetch_multi_with_patches\r\n\r\n3. One other detail about fetch_multi is that it has to take the multiplot limits into\r\naccount.  Multplot allows only 100 metrics per call, and has implicit limit of around\r\n7000 json characters. So my changes batch up each of graphite-apis fetch_multi() calls\r\ninto to the minimum number of multiplot calls that still abide by those limits.\r\n\r\n4. Now supports specificy specific submetric names in the config file, (rather\r\nthan just assuming \"average\", \"num_points\" or \"latest\" as we were before.\r\nThis required changes on both the search side, and the fetch_multi side.\r\nThis work was based on the changes suggested and created by Alex Weeks:\r\nhttps://github.com/aweeks\r\n\r\n5. Finally added a bunch of tests to get coverage of this file up to 93%:\r\nhttps://23282e1499f9e238db58e26751fc6d96be63cc17-www.googledrive.com/host/0B28IWTWKORCkfnlLYmljbW1NQ295NWhWbmtfbDJ1UXYwQm9xdlVuMjN0NFV1QTNmcDltY2M/x/blueflood.html\r\n","created_at":"2015-04-30T22:02:53Z","updated_at":"2015-05-08T21:28:02Z","closed_at":null,"merged_at":null,"merge_commit_sha":"b4a3560be919a176ddc07023154c98c83b09e2f3","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/rackerlabs/blueflood/pulls/460/commits","review_comments_url":"https://api.github.com/repos/rackerlabs/blueflood/pulls/460/comments","review_comment_url":"https://api.github.com/repos/rackerlabs/blueflood/pulls/comments{/number}","comments_url":"https://api.github.com/repos/rackerlabs/blueflood/issues/460/comments","statuses_url":"https://api.github.com/repos/rackerlabs/blueflood/statuses/c21dd403edc118ffe614324f143c433185acd5d3","head":{"label":"rackerlabs:george/add_fetch_multi2","ref":"george/add_fetch_multi2","sha":"c21dd403edc118ffe614324f143c433185acd5d3","user":{"login":"rackerlabs","id":84293,"avatar_url":"https://avatars.githubusercontent.com/u/84293?v=3","gravatar_id":"","url":"https://api.github.com/users/rackerlabs","html_url":"https://github.com/rackerlabs","followers_url":"https://api.github.com/users/rackerlabs/followers","following_url":"https://api.github.com/users/rackerlabs/following{/other_user}","gists_url":"https://api.github.com/users/rackerlabs/gists{/gist_id}","starred_url":"https://api.github.com/users/rackerlabs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rackerlabs/subscriptions","organizations_url":"https://api.github.com/users/rackerlabs/orgs","repos_url":"https://api.github.com/users/rackerlabs/repos","events_url":"https://api.github.com/users/rackerlabs/events{/privacy}","received_events_url":"https://api.github.com/users/rackerlabs/received_events","type":"Organization","site_admin":false},"repo":{"id":10081109,"name":"blueflood","full_name":"rackerlabs/blueflood","owner":{"login":"rackerlabs","id":84293,"avatar_url":"https://avatars.githubusercontent.com/u/84293?v=3","gravatar_id":"","url":"https://api.github.com/users/rackerlabs","html_url":"https://github.com/rackerlabs","followers_url":"https://api.github.com/users/rackerlabs/followers","following_url":"https://api.github.com/users/rackerlabs/following{/other_user}","gists_url":"https://api.github.com/users/rackerlabs/gists{/gist_id}","starred_url":"https://api.github.com/users/rackerlabs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rackerlabs/subscriptions","organizations_url":"https://api.github.com/users/rackerlabs/orgs","repos_url":"https://api.github.com/users/rackerlabs/repos","events_url":"https://api.github.com/users/rackerlabs/events{/privacy}","received_events_url":"https://api.github.com/users/rackerlabs/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/rackerlabs/blueflood","description":"A distributed system designed to ingest and process time series data","fork":false,"url":"https://api.github.com/repos/rackerlabs/blueflood","forks_url":"https://api.github.com/repos/rackerlabs/blueflood/forks","keys_url":"https://api.github.com/repos/rackerlabs/blueflood/keys{/key_id}","collaborators_url":"https://api.github.com/repos/rackerlabs/blueflood/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/rackerlabs/blueflood/teams","hooks_url":"https://api.github.com/repos/rackerlabs/blueflood/hooks","issue_events_url":"https://api.github.com/repos/rackerlabs/blueflood/issues/events{/number}","events_url":"https://api.github.com/repos/rackerlabs/blueflood/events","assignees_url":"https://api.github.com/repos/rackerlabs/blueflood/assignees{/user}","branches_url":"https://api.github.com/repos/rackerlabs/blueflood/branches{/branch}","tags_url":"https://api.github.com/repos/rackerlabs/blueflood/tags","blobs_url":"https://api.github.com/repos/rackerlabs/blueflood/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/rackerlabs/blueflood/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/rackerlabs/blueflood/git/refs{/sha}","trees_url":"https://api.github.com/repos/rackerlabs/blueflood/git/trees{/sha}","statuses_url":"https://api.github.com/repos/rackerlabs/blueflood/statuses/{sha}","languages_url":"https://api.github.com/repos/rackerlabs/blueflood/languages","stargazers_url":"https://api.github.com/repos/rackerlabs/blueflood/stargazers","contributors_url":"https://api.github.com/repos/rackerlabs/blueflood/contributors","subscribers_url":"https://api.github.com/repos/rackerlabs/blueflood/subscribers","subscription_url":"https://api.github.com/repos/rackerlabs/blueflood/subscription","commits_url":"https://api.github.com/repos/rackerlabs/blueflood/commits{/sha}","git_commits_url":"https://api.github.com/repos/rackerlabs/blueflood/git/commits{/sha}","comments_url":"https://api.github.com/repos/rackerlabs/blueflood/comments{/number}","issue_comment_url":"https://api.github.com/repos/rackerlabs/blueflood/issues/comments{/number}","contents_url":"https://api.github.com/repos/rackerlabs/blueflood/contents/{+path}","compare_url":"https://api.github.com/repos/rackerlabs/blueflood/compare/{base}...{head}","merges_url":"https://api.github.com/repos/rackerlabs/blueflood/merges","archive_url":"https://api.github.com/repos/rackerlabs/blueflood/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/rackerlabs/blueflood/downloads","issues_url":"https://api.github.com/repos/rackerlabs/blueflood/issues{/number}","pulls_url":"https://api.github.com/repos/rackerlabs/blueflood/pulls{/number}","milestones_url":"https://api.github.com/repos/rackerlabs/blueflood/milestones{/number}","notifications_url":"https://api.github.com/repos/rackerlabs/blueflood/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/rackerlabs/blueflood/labels{/name}","releases_url":"https://api.github.com/repos/rackerlabs/blueflood/releases{/id}","created_at":"2013-05-15T14:50:39Z","updated_at":"2015-05-07T23:55:03Z","pushed_at":"2015-05-08T17:41:43Z","git_url":"git://github.com/rackerlabs/blueflood.git","ssh_url":"git@github.com:rackerlabs/blueflood.git","clone_url":"https://github.com/rackerlabs/blueflood.git","svn_url":"https://github.com/rackerlabs/blueflood","homepage":"http://www.blueflood.io","size":11005,"stargazers_count":306,"watchers_count":306,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":41,"mirror_url":null,"open_issues_count":33,"forks":41,"open_issues":33,"watchers":306,"default_branch":"master"}},"base":{"label":"rackerlabs:master","ref":"master","sha":"acc39a6045440c03128b1c9684c071a229302cba","user":{"login":"rackerlabs","id":84293,"avatar_url":"https://avatars.githubusercontent.com/u/84293?v=3","gravatar_id":"","url":"https://api.github.com/users/rackerlabs","html_url":"https://github.com/rackerlabs","followers_url":"https://api.github.com/users/rackerlabs/followers","following_url":"https://api.github.com/users/rackerlabs/following{/other_user}","gists_url":"https://api.github.com/users/rackerlabs/gists{/gist_id}","starred_url":"https://api.github.com/users/rackerlabs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rackerlabs/subscriptions","organizations_url":"https://api.github.com/users/rackerlabs/orgs","repos_url":"https://api.github.com/users/rackerlabs/repos","events_url":"https://api.github.com/users/rackerlabs/events{/privacy}","received_events_url":"https://api.github.com/users/rackerlabs/received_events","type":"Organization","site_admin":false},"repo":{"id":10081109,"name":"blueflood","full_name":"rackerlabs/blueflood","owner":{"login":"rackerlabs","id":84293,"avatar_url":"https://avatars.githubusercontent.com/u/84293?v=3","gravatar_id":"","url":"https://api.github.com/users/rackerlabs","html_url":"https://github.com/rackerlabs","followers_url":"https://api.github.com/users/rackerlabs/followers","following_url":"https://api.github.com/users/rackerlabs/following{/other_user}","gists_url":"https://api.github.com/users/rackerlabs/gists{/gist_id}","starred_url":"https://api.github.com/users/rackerlabs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rackerlabs/subscriptions","organizations_url":"https://api.github.com/users/rackerlabs/orgs","repos_url":"https://api.github.com/users/rackerlabs/repos","events_url":"https://api.github.com/users/rackerlabs/events{/privacy}","received_events_url":"https://api.github.com/users/rackerlabs/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/rackerlabs/blueflood","description":"A distributed system designed to ingest and process time series data","fork":false,"url":"https://api.github.com/repos/rackerlabs/blueflood","forks_url":"https://api.github.com/repos/rackerlabs/blueflood/forks","keys_url":"https://api.github.com/repos/rackerlabs/blueflood/keys{/key_id}","collaborators_url":"https://api.github.com/repos/rackerlabs/blueflood/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/rackerlabs/blueflood/teams","hooks_url":"https://api.github.com/repos/rackerlabs/blueflood/hooks","issue_events_url":"https://api.github.com/repos/rackerlabs/blueflood/issues/events{/number}","events_url":"https://api.github.com/repos/rackerlabs/blueflood/events","assignees_url":"https://api.github.com/repos/rackerlabs/blueflood/assignees{/user}","branches_url":"https://api.github.com/repos/rackerlabs/blueflood/branches{/branch}","tags_url":"https://api.github.com/repos/rackerlabs/blueflood/tags","blobs_url":"https://api.github.com/repos/rackerlabs/blueflood/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/rackerlabs/blueflood/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/rackerlabs/blueflood/git/refs{/sha}","trees_url":"https://api.github.com/repos/rackerlabs/blueflood/git/trees{/sha}","statuses_url":"https://api.github.com/repos/rackerlabs/blueflood/statuses/{sha}","languages_url":"https://api.github.com/repos/rackerlabs/blueflood/languages","stargazers_url":"https://api.github.com/repos/rackerlabs/blueflood/stargazers","contributors_url":"https://api.github.com/repos/rackerlabs/blueflood/contributors","subscribers_url":"https://api.github.com/repos/rackerlabs/blueflood/subscribers","subscription_url":"https://api.github.com/repos/rackerlabs/blueflood/subscription","commits_url":"https://api.github.com/repos/rackerlabs/blueflood/commits{/sha}","git_commits_url":"https://api.github.com/repos/rackerlabs/blueflood/git/commits{/sha}","comments_url":"https://api.github.com/repos/rackerlabs/blueflood/comments{/number}","issue_comment_url":"https://api.github.com/repos/rackerlabs/blueflood/issues/comments{/number}","contents_url":"https://api.github.com/repos/rackerlabs/blueflood/contents/{+path}","compare_url":"https://api.github.com/repos/rackerlabs/blueflood/compare/{base}...{head}","merges_url":"https://api.github.com/repos/rackerlabs/blueflood/merges","archive_url":"https://api.github.com/repos/rackerlabs/blueflood/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/rackerlabs/blueflood/downloads","issues_url":"https://api.github.com/repos/rackerlabs/blueflood/issues{/number}","pulls_url":"https://api.github.com/repos/rackerlabs/blueflood/pulls{/number}","milestones_url":"https://api.github.com/repos/rackerlabs/blueflood/milestones{/number}","notifications_url":"https://api.github.com/repos/rackerlabs/blueflood/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/rackerlabs/blueflood/labels{/name}","releases_url":"https://api.github.com/repos/rackerlabs/blueflood/releases{/id}","created_at":"2013-05-15T14:50:39Z","updated_at":"2015-05-07T23:55:03Z","pushed_at":"2015-05-08T17:41:43Z","git_url":"git://github.com/rackerlabs/blueflood.git","ssh_url":"git@github.com:rackerlabs/blueflood.git","clone_url":"https://github.com/rackerlabs/blueflood.git","svn_url":"https://github.com/rackerlabs/blueflood","homepage":"http://www.blueflood.io","size":11005,"stargazers_count":306,"watchers_count":306,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":41,"mirror_url":null,"open_issues_count":33,"forks":41,"open_issues":33,"watchers":306,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/rackerlabs/blueflood/pulls/460"},"html":{"href":"https://github.com/rackerlabs/blueflood/pull/460"},"issue":{"href":"https://api.github.com/repos/rackerlabs/blueflood/issues/460"},"comments":{"href":"https://api.github.com/repos/rackerlabs/blueflood/issues/460/comments"},"review_comments":{"href":"https://api.github.com/repos/rackerlabs/blueflood/pulls/460/comments"},"review_comment":{"href":"https://api.github.com/repos/rackerlabs/blueflood/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/rackerlabs/blueflood/pulls/460/commits"},"statuses":{"href":"https://api.github.com/repos/rackerlabs/blueflood/statuses/c21dd403edc118ffe614324f143c433185acd5d3"}}}}