{"action":"created","comment":{"url":"https://api.github.com/repos/Wikia/sparrow/pulls/comments/30890063","id":30890063,"diff_hunk":"@@ -11,18 +13,100 @@\n \n class ProcessResponses(Action):\n     REQUIRED_PARAMS = (\n-        'responses',\n+        'results',\n     )\n \n-    def run(self):\n-        logger.info('Starting processing {0} results'.format(len(self.params['responses'])))\n-\n-        metrics = [float(response.headers['X-Backend-Response-Time']) for response in self.params['responses']]\n-        self.result['response_time'] = {\n-            'mean': numpy.mean(metrics),\n-            'median': numpy.median(metrics),\n-            'lowest': numpy.min(metrics),\n-            'highest': numpy.max(metrics),\n+    __PROFILER_REGEXP = re.compile(r'^\\s*([\\d\\.]+\\%)\\s+([\\d\\.]+)\\s+(\\d+)\\s+\\-\\s+([^\\s].*[^\\s])\\s*$')\n+    __MEMCACHE_REGEXP = re.compile(r'MWMemcached::get.*!(HIT|MISS|DUPE)')\n+\n+    def _parse_metrics(self, raw_data):\n+        metrics = {\n+            'server_time': 0.0,\n+            'query_time': 0.0,\n+            'query_master': 0,\n+            'query_slave': 0,\n+            'memc_time': 0.0,\n+            'memc_misses': 0,\n+            'memc_hits': 0,\n+            'memc_dupes': 0\n+        }\n+\n+        for line in raw_data.splitlines():\n+            match = self.__PROFILER_REGEXP.match(line)\n+\n+            if match is not None:\n+                (percent, time, count, name) = match.groups()\n+\n+                percent = float(percent.strip('%'))\n+                time = float(time)\n+                count = int(count)\n+\n+                if name == '-total':\n+                    metrics['server_time'] += time\n+                elif name == 'DatabaseBase::query' or name == 'DatabaseBase::query-master':\n+                    is_master = name == 'DatabaseBase::query-master'\n+                    metrics['query_time'] += time\n+\n+                    if is_master:\n+                        metrics['query_master'] += count\n+                    else:\n+                        metrics['query_slave'] += count\n+                elif name == 'MWMemcached::get':\n+                    metrics['memc_time'] += time\n+                else:\n+                    match = re.match(self.__MEMCACHE_REGEXP, name)\n+\n+                    if match is not None:\n+                        memc_type = match.group(1)\n+                        if memc_type == 'HIT':\n+                            metrics['memc_hits'] += count\n+                        elif memc_type == 'MISS':\n+                            metrics['memc_misses'] += count\n+                        elif memc_type == 'DUPE':\n+                            metrics['memc_dupes'] += count\n+        return metrics\n+\n+    @staticmethod\n+    def _calculate_stats(values):\n+        return {\n+            'count': int(len(values)),\n+            '5th_percentile': float(numpy.percentile(values, 5)),\n+            '50th_percentile': float(numpy.percentile(values, 50)),\n+            '90th_percentile': float(numpy.percentile(values, 90)),\n+            '95th_percentile': float(numpy.percentile(values, 95)),\n+            'std': float(numpy.std(values)),\n+            'mean': float(numpy.mean(values)),\n+            'median': float(numpy.median(values)),\n+            'lowest': float(numpy.min(values)),\n+            'highest': float(numpy.max(values)),\n         }\n \n+    def run(self):\n+        logger.info('Starting processing results')","path":"apps/testrunner/actions/process_results.py","position":92,"original_position":92,"commit_id":"e44afbf56d7bb9198e28e09ce630bdee0d1596c2","original_commit_id":"e44afbf56d7bb9198e28e09ce630bdee0d1596c2","user":{"login":"harnash","id":783242,"avatar_url":"https://avatars.githubusercontent.com/u/783242?v=3","gravatar_id":"","url":"https://api.github.com/users/harnash","html_url":"https://github.com/harnash","followers_url":"https://api.github.com/users/harnash/followers","following_url":"https://api.github.com/users/harnash/following{/other_user}","gists_url":"https://api.github.com/users/harnash/gists{/gist_id}","starred_url":"https://api.github.com/users/harnash/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/harnash/subscriptions","organizations_url":"https://api.github.com/users/harnash/orgs","repos_url":"https://api.github.com/users/harnash/repos","events_url":"https://api.github.com/users/harnash/events{/privacy}","received_events_url":"https://api.github.com/users/harnash/received_events","type":"User","site_admin":false},"body":"It should be obvious for now. We have info about each task being processed. After we do refactor those classes we should come up with better solution. Right now it is not worth since it will be rewritten anyway.","created_at":"2015-05-22T12:21:43Z","updated_at":"2015-05-22T12:21:43Z","html_url":"https://github.com/Wikia/sparrow/pull/20#discussion_r30890063","pull_request_url":"https://api.github.com/repos/Wikia/sparrow/pulls/20","_links":{"self":{"href":"https://api.github.com/repos/Wikia/sparrow/pulls/comments/30890063"},"html":{"href":"https://github.com/Wikia/sparrow/pull/20#discussion_r30890063"},"pull_request":{"href":"https://api.github.com/repos/Wikia/sparrow/pulls/20"}}},"pull_request":{"url":"https://api.github.com/repos/Wikia/sparrow/pulls/20","id":36019251,"html_url":"https://github.com/Wikia/sparrow/pull/20","diff_url":"https://github.com/Wikia/sparrow/pull/20.diff","patch_url":"https://github.com/Wikia/sparrow/pull/20.patch","issue_url":"https://api.github.com/repos/Wikia/sparrow/issues/20","number":20,"state":"open","locked":false,"title":"Backend metrics","user":{"login":"harnash","id":783242,"avatar_url":"https://avatars.githubusercontent.com/u/783242?v=3","gravatar_id":"","url":"https://api.github.com/users/harnash","html_url":"https://github.com/harnash","followers_url":"https://api.github.com/users/harnash/followers","following_url":"https://api.github.com/users/harnash/following{/other_user}","gists_url":"https://api.github.com/users/harnash/gists{/gist_id}","starred_url":"https://api.github.com/users/harnash/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/harnash/subscriptions","organizations_url":"https://api.github.com/users/harnash/orgs","repos_url":"https://api.github.com/users/harnash/repos","events_url":"https://api.github.com/users/harnash/events{/privacy}","received_events_url":"https://api.github.com/users/harnash/received_events","type":"User","site_admin":false},"body":"Added collecting of backend metrics from Media Wiki. It runs a page with forceprofile=1 and parses resulting data.","created_at":"2015-05-22T10:40:48Z","updated_at":"2015-05-22T12:21:43Z","closed_at":null,"merged_at":null,"merge_commit_sha":"8899382d6999b371f6e07866fd636c90afad942f","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/Wikia/sparrow/pulls/20/commits","review_comments_url":"https://api.github.com/repos/Wikia/sparrow/pulls/20/comments","review_comment_url":"https://api.github.com/repos/Wikia/sparrow/pulls/comments{/number}","comments_url":"https://api.github.com/repos/Wikia/sparrow/issues/20/comments","statuses_url":"https://api.github.com/repos/Wikia/sparrow/statuses/e44afbf56d7bb9198e28e09ce630bdee0d1596c2","head":{"label":"Wikia:backend_metrics","ref":"backend_metrics","sha":"e44afbf56d7bb9198e28e09ce630bdee0d1596c2","user":{"login":"Wikia","id":1171011,"avatar_url":"https://avatars.githubusercontent.com/u/1171011?v=3","gravatar_id":"","url":"https://api.github.com/users/Wikia","html_url":"https://github.com/Wikia","followers_url":"https://api.github.com/users/Wikia/followers","following_url":"https://api.github.com/users/Wikia/following{/other_user}","gists_url":"https://api.github.com/users/Wikia/gists{/gist_id}","starred_url":"https://api.github.com/users/Wikia/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Wikia/subscriptions","organizations_url":"https://api.github.com/users/Wikia/orgs","repos_url":"https://api.github.com/users/Wikia/repos","events_url":"https://api.github.com/users/Wikia/events{/privacy}","received_events_url":"https://api.github.com/users/Wikia/received_events","type":"Organization","site_admin":false},"repo":{"id":34334701,"name":"sparrow","full_name":"Wikia/sparrow","owner":{"login":"Wikia","id":1171011,"avatar_url":"https://avatars.githubusercontent.com/u/1171011?v=3","gravatar_id":"","url":"https://api.github.com/users/Wikia","html_url":"https://github.com/Wikia","followers_url":"https://api.github.com/users/Wikia/followers","following_url":"https://api.github.com/users/Wikia/following{/other_user}","gists_url":"https://api.github.com/users/Wikia/gists{/gist_id}","starred_url":"https://api.github.com/users/Wikia/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Wikia/subscriptions","organizations_url":"https://api.github.com/users/Wikia/orgs","repos_url":"https://api.github.com/users/Wikia/repos","events_url":"https://api.github.com/users/Wikia/events{/privacy}","received_events_url":"https://api.github.com/users/Wikia/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/Wikia/sparrow","description":"","fork":false,"url":"https://api.github.com/repos/Wikia/sparrow","forks_url":"https://api.github.com/repos/Wikia/sparrow/forks","keys_url":"https://api.github.com/repos/Wikia/sparrow/keys{/key_id}","collaborators_url":"https://api.github.com/repos/Wikia/sparrow/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/Wikia/sparrow/teams","hooks_url":"https://api.github.com/repos/Wikia/sparrow/hooks","issue_events_url":"https://api.github.com/repos/Wikia/sparrow/issues/events{/number}","events_url":"https://api.github.com/repos/Wikia/sparrow/events","assignees_url":"https://api.github.com/repos/Wikia/sparrow/assignees{/user}","branches_url":"https://api.github.com/repos/Wikia/sparrow/branches{/branch}","tags_url":"https://api.github.com/repos/Wikia/sparrow/tags","blobs_url":"https://api.github.com/repos/Wikia/sparrow/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/Wikia/sparrow/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/Wikia/sparrow/git/refs{/sha}","trees_url":"https://api.github.com/repos/Wikia/sparrow/git/trees{/sha}","statuses_url":"https://api.github.com/repos/Wikia/sparrow/statuses/{sha}","languages_url":"https://api.github.com/repos/Wikia/sparrow/languages","stargazers_url":"https://api.github.com/repos/Wikia/sparrow/stargazers","contributors_url":"https://api.github.com/repos/Wikia/sparrow/contributors","subscribers_url":"https://api.github.com/repos/Wikia/sparrow/subscribers","subscription_url":"https://api.github.com/repos/Wikia/sparrow/subscription","commits_url":"https://api.github.com/repos/Wikia/sparrow/commits{/sha}","git_commits_url":"https://api.github.com/repos/Wikia/sparrow/git/commits{/sha}","comments_url":"https://api.github.com/repos/Wikia/sparrow/comments{/number}","issue_comment_url":"https://api.github.com/repos/Wikia/sparrow/issues/comments{/number}","contents_url":"https://api.github.com/repos/Wikia/sparrow/contents/{+path}","compare_url":"https://api.github.com/repos/Wikia/sparrow/compare/{base}...{head}","merges_url":"https://api.github.com/repos/Wikia/sparrow/merges","archive_url":"https://api.github.com/repos/Wikia/sparrow/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/Wikia/sparrow/downloads","issues_url":"https://api.github.com/repos/Wikia/sparrow/issues{/number}","pulls_url":"https://api.github.com/repos/Wikia/sparrow/pulls{/number}","milestones_url":"https://api.github.com/repos/Wikia/sparrow/milestones{/number}","notifications_url":"https://api.github.com/repos/Wikia/sparrow/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/Wikia/sparrow/labels{/name}","releases_url":"https://api.github.com/repos/Wikia/sparrow/releases{/id}","created_at":"2015-04-21T15:12:14Z","updated_at":"2015-05-12T09:55:38Z","pushed_at":"2015-05-22T11:11:41Z","git_url":"git://github.com/Wikia/sparrow.git","ssh_url":"git@github.com:Wikia/sparrow.git","clone_url":"https://github.com/Wikia/sparrow.git","svn_url":"https://github.com/Wikia/sparrow","homepage":null,"size":848,"stargazers_count":1,"watchers_count":1,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":2,"forks":0,"open_issues":2,"watchers":1,"default_branch":"master"}},"base":{"label":"Wikia:master","ref":"master","sha":"898490e2771be63351b414def82c55bbba10cc79","user":{"login":"Wikia","id":1171011,"avatar_url":"https://avatars.githubusercontent.com/u/1171011?v=3","gravatar_id":"","url":"https://api.github.com/users/Wikia","html_url":"https://github.com/Wikia","followers_url":"https://api.github.com/users/Wikia/followers","following_url":"https://api.github.com/users/Wikia/following{/other_user}","gists_url":"https://api.github.com/users/Wikia/gists{/gist_id}","starred_url":"https://api.github.com/users/Wikia/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Wikia/subscriptions","organizations_url":"https://api.github.com/users/Wikia/orgs","repos_url":"https://api.github.com/users/Wikia/repos","events_url":"https://api.github.com/users/Wikia/events{/privacy}","received_events_url":"https://api.github.com/users/Wikia/received_events","type":"Organization","site_admin":false},"repo":{"id":34334701,"name":"sparrow","full_name":"Wikia/sparrow","owner":{"login":"Wikia","id":1171011,"avatar_url":"https://avatars.githubusercontent.com/u/1171011?v=3","gravatar_id":"","url":"https://api.github.com/users/Wikia","html_url":"https://github.com/Wikia","followers_url":"https://api.github.com/users/Wikia/followers","following_url":"https://api.github.com/users/Wikia/following{/other_user}","gists_url":"https://api.github.com/users/Wikia/gists{/gist_id}","starred_url":"https://api.github.com/users/Wikia/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Wikia/subscriptions","organizations_url":"https://api.github.com/users/Wikia/orgs","repos_url":"https://api.github.com/users/Wikia/repos","events_url":"https://api.github.com/users/Wikia/events{/privacy}","received_events_url":"https://api.github.com/users/Wikia/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/Wikia/sparrow","description":"","fork":false,"url":"https://api.github.com/repos/Wikia/sparrow","forks_url":"https://api.github.com/repos/Wikia/sparrow/forks","keys_url":"https://api.github.com/repos/Wikia/sparrow/keys{/key_id}","collaborators_url":"https://api.github.com/repos/Wikia/sparrow/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/Wikia/sparrow/teams","hooks_url":"https://api.github.com/repos/Wikia/sparrow/hooks","issue_events_url":"https://api.github.com/repos/Wikia/sparrow/issues/events{/number}","events_url":"https://api.github.com/repos/Wikia/sparrow/events","assignees_url":"https://api.github.com/repos/Wikia/sparrow/assignees{/user}","branches_url":"https://api.github.com/repos/Wikia/sparrow/branches{/branch}","tags_url":"https://api.github.com/repos/Wikia/sparrow/tags","blobs_url":"https://api.github.com/repos/Wikia/sparrow/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/Wikia/sparrow/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/Wikia/sparrow/git/refs{/sha}","trees_url":"https://api.github.com/repos/Wikia/sparrow/git/trees{/sha}","statuses_url":"https://api.github.com/repos/Wikia/sparrow/statuses/{sha}","languages_url":"https://api.github.com/repos/Wikia/sparrow/languages","stargazers_url":"https://api.github.com/repos/Wikia/sparrow/stargazers","contributors_url":"https://api.github.com/repos/Wikia/sparrow/contributors","subscribers_url":"https://api.github.com/repos/Wikia/sparrow/subscribers","subscription_url":"https://api.github.com/repos/Wikia/sparrow/subscription","commits_url":"https://api.github.com/repos/Wikia/sparrow/commits{/sha}","git_commits_url":"https://api.github.com/repos/Wikia/sparrow/git/commits{/sha}","comments_url":"https://api.github.com/repos/Wikia/sparrow/comments{/number}","issue_comment_url":"https://api.github.com/repos/Wikia/sparrow/issues/comments{/number}","contents_url":"https://api.github.com/repos/Wikia/sparrow/contents/{+path}","compare_url":"https://api.github.com/repos/Wikia/sparrow/compare/{base}...{head}","merges_url":"https://api.github.com/repos/Wikia/sparrow/merges","archive_url":"https://api.github.com/repos/Wikia/sparrow/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/Wikia/sparrow/downloads","issues_url":"https://api.github.com/repos/Wikia/sparrow/issues{/number}","pulls_url":"https://api.github.com/repos/Wikia/sparrow/pulls{/number}","milestones_url":"https://api.github.com/repos/Wikia/sparrow/milestones{/number}","notifications_url":"https://api.github.com/repos/Wikia/sparrow/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/Wikia/sparrow/labels{/name}","releases_url":"https://api.github.com/repos/Wikia/sparrow/releases{/id}","created_at":"2015-04-21T15:12:14Z","updated_at":"2015-05-12T09:55:38Z","pushed_at":"2015-05-22T11:11:41Z","git_url":"git://github.com/Wikia/sparrow.git","ssh_url":"git@github.com:Wikia/sparrow.git","clone_url":"https://github.com/Wikia/sparrow.git","svn_url":"https://github.com/Wikia/sparrow","homepage":null,"size":848,"stargazers_count":1,"watchers_count":1,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":2,"forks":0,"open_issues":2,"watchers":1,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/Wikia/sparrow/pulls/20"},"html":{"href":"https://github.com/Wikia/sparrow/pull/20"},"issue":{"href":"https://api.github.com/repos/Wikia/sparrow/issues/20"},"comments":{"href":"https://api.github.com/repos/Wikia/sparrow/issues/20/comments"},"review_comments":{"href":"https://api.github.com/repos/Wikia/sparrow/pulls/20/comments"},"review_comment":{"href":"https://api.github.com/repos/Wikia/sparrow/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/Wikia/sparrow/pulls/20/commits"},"statuses":{"href":"https://api.github.com/repos/Wikia/sparrow/statuses/e44afbf56d7bb9198e28e09ce630bdee0d1596c2"}}}}