{"action":"created","comment":{"url":"https://api.github.com/repos/mozilla/olympia/pulls/comments/35406241","id":35406241,"diff_hunk":"@@ -0,0 +1,125 @@\n+import itertools\n+import json\n+import sys\n+\n+ACTION_CONTEXT = 'context'\n+ACTION_COUNT = 'count'\n+ACTIONS = (ACTION_CONTEXT, ACTION_COUNT)\n+\n+UNLISTED_ADDONS = set()\n+\n+\n+def parse_validations(results):\n+    return (json.loads(result) for result in results)\n+\n+\n+def unlisted_validations(results, unlisted_addons=None):\n+    if unlisted_addons is None:\n+        unlisted_addons = UNLISTED_ADDONS\n+    return (result\n+            for result in results\n+            if (not result['metadata'].get('listed', True)\n+                or result['metadata'].get('id') in unlisted_addons))\n+\n+\n+def severe_validations(results):\n+    return (result\n+            for result in results\n+            if (result['signing_summary']['high'] > 0 or\n+                result['signing_summary']['medium'] > 0 or\n+                result['signing_summary']['low'] > 0))\n+\n+\n+def error_messages(results):\n+    return ({'addon': result['metadata'].get('id', 'unknown-addon'),\n+             'message_id': '.'.join(message['id']),\n+             'context': message['context']}\n+            for result in results\n+            for message in result['messages']\n+            if 'signing_severity' in message)\n+\n+\n+def sort_by_message(results):\n+    return sorted(results, key=lambda r: r['message_id'])\n+\n+\n+def group_by_message(results):\n+    return itertools.groupby(results, lambda r: r['message_id'])\n+\n+\n+def extract_error_results(results):\n+    for error, messages in results:\n+        all_messages = list(messages)\n+        yield {\n+            'error': error,\n+            'total': len(all_messages),\n+            'unique': len(set(msg['addon'] for msg in all_messages)),\n+            'contexts': [msg['context'] for msg in all_messages],\n+        }\n+\n+\n+def sort_results_by_unique(results):\n+    return sorted(results, reverse=True, key=lambda r: r['unique'])\n+\n+\n+def format_error_count(results):\n+    return ('{error} {total} {unique}'.format(**result)\n+            for result in results)\n+\n+\n+def format_contexts(results):\n+    for result in results:\n+        for context in result['contexts']:\n+            yield json.dumps({\n+                'error': result['error'],\n+                'context': context,\n+            })\n+\n+\n+def parse_unlisted_addons():\n+    global UNLISTED_ADDONS\n+    with open('validations/unlisted-addons.txt') as f:\n+        UNLISTED_ADDONS = set(guid.strip() for guid in f)\n+\n+\n+def main(action):\n+    pipeline = [\n+        parse_validations,\n+        unlisted_validations,\n+        severe_validations,\n+        error_messages,\n+        sort_by_message,\n+        group_by_message,\n+        extract_error_results,\n+        sort_results_by_unique,\n+    ]\n+\n+    if action == ACTION_CONTEXT:\n+        pipeline.append(lambda results: itertools.islice(results, 5))\n+        pipeline.append(format_contexts)\n+    elif action == ACTION_COUNT:\n+        pipeline.append(format_error_count)\n+        parse_unlisted_addons()\n+    else:\n+        raise ValueError('{0} is not a valid action'.format(action))\n+\n+    # Read from STDIN.\n+    val = sys.stdin\n+\n+    # Process through the pipeline.\n+    for fn in pipeline:\n+        val = fn(val)","path":"scripts/validations.py","position":111,"original_position":111,"commit_id":"4cb82e39635e5509862fa16c5e3de9c9efcc9aaf","original_commit_id":"4cb82e39635e5509862fa16c5e3de9c9efcc9aaf","user":{"login":"magopian","id":167767,"avatar_url":"https://avatars.githubusercontent.com/u/167767?v=3","gravatar_id":"","url":"https://api.github.com/users/magopian","html_url":"https://github.com/magopian","followers_url":"https://api.github.com/users/magopian/followers","following_url":"https://api.github.com/users/magopian/following{/other_user}","gists_url":"https://api.github.com/users/magopian/gists{/gist_id}","starred_url":"https://api.github.com/users/magopian/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/magopian/subscriptions","organizations_url":"https://api.github.com/users/magopian/orgs","repos_url":"https://api.github.com/users/magopian/repos","events_url":"https://api.github.com/users/magopian/events{/privacy}","received_events_url":"https://api.github.com/users/magopian/received_events","type":"User","site_admin":false},"body":"<3 functional programming <3","created_at":"2015-07-24T09:00:35Z","updated_at":"2015-07-24T09:00:35Z","html_url":"https://github.com/mozilla/olympia/pull/613#discussion_r35406241","pull_request_url":"https://api.github.com/repos/mozilla/olympia/pulls/613","_links":{"self":{"href":"https://api.github.com/repos/mozilla/olympia/pulls/comments/35406241"},"html":{"href":"https://github.com/mozilla/olympia/pull/613#discussion_r35406241"},"pull_request":{"href":"https://api.github.com/repos/mozilla/olympia/pulls/613"}}},"pull_request":{"url":"https://api.github.com/repos/mozilla/olympia/pulls/613","id":40613356,"html_url":"https://github.com/mozilla/olympia/pull/613","diff_url":"https://github.com/mozilla/olympia/pull/613.diff","patch_url":"https://github.com/mozilla/olympia/pull/613.patch","issue_url":"https://api.github.com/repos/mozilla/olympia/issues/613","number":613,"state":"open","locked":false,"title":"Scripts to analyze validation results (bug 1183396)","user":{"login":"mstriemer","id":211578,"avatar_url":"https://avatars.githubusercontent.com/u/211578?v=3","gravatar_id":"","url":"https://api.github.com/users/mstriemer","html_url":"https://github.com/mstriemer","followers_url":"https://api.github.com/users/mstriemer/followers","following_url":"https://api.github.com/users/mstriemer/following{/other_user}","gists_url":"https://api.github.com/users/mstriemer/gists{/gist_id}","starred_url":"https://api.github.com/users/mstriemer/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mstriemer/subscriptions","organizations_url":"https://api.github.com/users/mstriemer/orgs","repos_url":"https://api.github.com/users/mstriemer/repos","events_url":"https://api.github.com/users/mstriemer/events{/privacy}","received_events_url":"https://api.github.com/users/mstriemer/received_events","type":"User","site_admin":false},"body":"https://bugzilla.mozilla.org/show_bug.cgi?id=1183396\r\n\r\nScripts to download validation results by day and process the results (find common errors and retrieve contexts for errors)","created_at":"2015-07-22T21:23:09Z","updated_at":"2015-07-24T09:00:35Z","closed_at":null,"merged_at":null,"merge_commit_sha":"78ef392f2e72e960c3c121dc3efa3bce6093cd0e","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/mozilla/olympia/pulls/613/commits","review_comments_url":"https://api.github.com/repos/mozilla/olympia/pulls/613/comments","review_comment_url":"https://api.github.com/repos/mozilla/olympia/pulls/comments{/number}","comments_url":"https://api.github.com/repos/mozilla/olympia/issues/613/comments","statuses_url":"https://api.github.com/repos/mozilla/olympia/statuses/4cb82e39635e5509862fa16c5e3de9c9efcc9aaf","head":{"label":"mstriemer:analyze-validation-data-1183396","ref":"analyze-validation-data-1183396","sha":"4cb82e39635e5509862fa16c5e3de9c9efcc9aaf","user":{"login":"mstriemer","id":211578,"avatar_url":"https://avatars.githubusercontent.com/u/211578?v=3","gravatar_id":"","url":"https://api.github.com/users/mstriemer","html_url":"https://github.com/mstriemer","followers_url":"https://api.github.com/users/mstriemer/followers","following_url":"https://api.github.com/users/mstriemer/following{/other_user}","gists_url":"https://api.github.com/users/mstriemer/gists{/gist_id}","starred_url":"https://api.github.com/users/mstriemer/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mstriemer/subscriptions","organizations_url":"https://api.github.com/users/mstriemer/orgs","repos_url":"https://api.github.com/users/mstriemer/repos","events_url":"https://api.github.com/users/mstriemer/events{/privacy}","received_events_url":"https://api.github.com/users/mstriemer/received_events","type":"User","site_admin":false},"repo":{"id":18221347,"name":"olympia","full_name":"mstriemer/olympia","owner":{"login":"mstriemer","id":211578,"avatar_url":"https://avatars.githubusercontent.com/u/211578?v=3","gravatar_id":"","url":"https://api.github.com/users/mstriemer","html_url":"https://github.com/mstriemer","followers_url":"https://api.github.com/users/mstriemer/followers","following_url":"https://api.github.com/users/mstriemer/following{/other_user}","gists_url":"https://api.github.com/users/mstriemer/gists{/gist_id}","starred_url":"https://api.github.com/users/mstriemer/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mstriemer/subscriptions","organizations_url":"https://api.github.com/users/mstriemer/orgs","repos_url":"https://api.github.com/users/mstriemer/repos","events_url":"https://api.github.com/users/mstriemer/events{/privacy}","received_events_url":"https://api.github.com/users/mstriemer/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/mstriemer/olympia","description":"AMO, reloaded.","fork":true,"url":"https://api.github.com/repos/mstriemer/olympia","forks_url":"https://api.github.com/repos/mstriemer/olympia/forks","keys_url":"https://api.github.com/repos/mstriemer/olympia/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mstriemer/olympia/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mstriemer/olympia/teams","hooks_url":"https://api.github.com/repos/mstriemer/olympia/hooks","issue_events_url":"https://api.github.com/repos/mstriemer/olympia/issues/events{/number}","events_url":"https://api.github.com/repos/mstriemer/olympia/events","assignees_url":"https://api.github.com/repos/mstriemer/olympia/assignees{/user}","branches_url":"https://api.github.com/repos/mstriemer/olympia/branches{/branch}","tags_url":"https://api.github.com/repos/mstriemer/olympia/tags","blobs_url":"https://api.github.com/repos/mstriemer/olympia/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mstriemer/olympia/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mstriemer/olympia/git/refs{/sha}","trees_url":"https://api.github.com/repos/mstriemer/olympia/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mstriemer/olympia/statuses/{sha}","languages_url":"https://api.github.com/repos/mstriemer/olympia/languages","stargazers_url":"https://api.github.com/repos/mstriemer/olympia/stargazers","contributors_url":"https://api.github.com/repos/mstriemer/olympia/contributors","subscribers_url":"https://api.github.com/repos/mstriemer/olympia/subscribers","subscription_url":"https://api.github.com/repos/mstriemer/olympia/subscription","commits_url":"https://api.github.com/repos/mstriemer/olympia/commits{/sha}","git_commits_url":"https://api.github.com/repos/mstriemer/olympia/git/commits{/sha}","comments_url":"https://api.github.com/repos/mstriemer/olympia/comments{/number}","issue_comment_url":"https://api.github.com/repos/mstriemer/olympia/issues/comments{/number}","contents_url":"https://api.github.com/repos/mstriemer/olympia/contents/{+path}","compare_url":"https://api.github.com/repos/mstriemer/olympia/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mstriemer/olympia/merges","archive_url":"https://api.github.com/repos/mstriemer/olympia/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mstriemer/olympia/downloads","issues_url":"https://api.github.com/repos/mstriemer/olympia/issues{/number}","pulls_url":"https://api.github.com/repos/mstriemer/olympia/pulls{/number}","milestones_url":"https://api.github.com/repos/mstriemer/olympia/milestones{/number}","notifications_url":"https://api.github.com/repos/mstriemer/olympia/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mstriemer/olympia/labels{/name}","releases_url":"https://api.github.com/repos/mstriemer/olympia/releases{/id}","created_at":"2014-03-28T17:58:28Z","updated_at":"2014-06-24T22:05:02Z","pushed_at":"2015-07-23T19:16:43Z","git_url":"git://github.com/mstriemer/olympia.git","ssh_url":"git@github.com:mstriemer/olympia.git","clone_url":"https://github.com/mstriemer/olympia.git","svn_url":"https://github.com/mstriemer/olympia","homepage":"","size":392216,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"mozilla:master","ref":"master","sha":"80f5a642f6f8f9d9d6d85ee1a58469bdc8ff40b7","user":{"login":"mozilla","id":131524,"avatar_url":"https://avatars.githubusercontent.com/u/131524?v=3","gravatar_id":"","url":"https://api.github.com/users/mozilla","html_url":"https://github.com/mozilla","followers_url":"https://api.github.com/users/mozilla/followers","following_url":"https://api.github.com/users/mozilla/following{/other_user}","gists_url":"https://api.github.com/users/mozilla/gists{/gist_id}","starred_url":"https://api.github.com/users/mozilla/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mozilla/subscriptions","organizations_url":"https://api.github.com/users/mozilla/orgs","repos_url":"https://api.github.com/users/mozilla/repos","events_url":"https://api.github.com/users/mozilla/events{/privacy}","received_events_url":"https://api.github.com/users/mozilla/received_events","type":"Organization","site_admin":false},"repo":{"id":16416867,"name":"olympia","full_name":"mozilla/olympia","owner":{"login":"mozilla","id":131524,"avatar_url":"https://avatars.githubusercontent.com/u/131524?v=3","gravatar_id":"","url":"https://api.github.com/users/mozilla","html_url":"https://github.com/mozilla","followers_url":"https://api.github.com/users/mozilla/followers","following_url":"https://api.github.com/users/mozilla/following{/other_user}","gists_url":"https://api.github.com/users/mozilla/gists{/gist_id}","starred_url":"https://api.github.com/users/mozilla/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mozilla/subscriptions","organizations_url":"https://api.github.com/users/mozilla/orgs","repos_url":"https://api.github.com/users/mozilla/repos","events_url":"https://api.github.com/users/mozilla/events{/privacy}","received_events_url":"https://api.github.com/users/mozilla/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/mozilla/olympia","description":"addons.mozilla.org, reloaded.","fork":false,"url":"https://api.github.com/repos/mozilla/olympia","forks_url":"https://api.github.com/repos/mozilla/olympia/forks","keys_url":"https://api.github.com/repos/mozilla/olympia/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mozilla/olympia/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mozilla/olympia/teams","hooks_url":"https://api.github.com/repos/mozilla/olympia/hooks","issue_events_url":"https://api.github.com/repos/mozilla/olympia/issues/events{/number}","events_url":"https://api.github.com/repos/mozilla/olympia/events","assignees_url":"https://api.github.com/repos/mozilla/olympia/assignees{/user}","branches_url":"https://api.github.com/repos/mozilla/olympia/branches{/branch}","tags_url":"https://api.github.com/repos/mozilla/olympia/tags","blobs_url":"https://api.github.com/repos/mozilla/olympia/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mozilla/olympia/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mozilla/olympia/git/refs{/sha}","trees_url":"https://api.github.com/repos/mozilla/olympia/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mozilla/olympia/statuses/{sha}","languages_url":"https://api.github.com/repos/mozilla/olympia/languages","stargazers_url":"https://api.github.com/repos/mozilla/olympia/stargazers","contributors_url":"https://api.github.com/repos/mozilla/olympia/contributors","subscribers_url":"https://api.github.com/repos/mozilla/olympia/subscribers","subscription_url":"https://api.github.com/repos/mozilla/olympia/subscription","commits_url":"https://api.github.com/repos/mozilla/olympia/commits{/sha}","git_commits_url":"https://api.github.com/repos/mozilla/olympia/git/commits{/sha}","comments_url":"https://api.github.com/repos/mozilla/olympia/comments{/number}","issue_comment_url":"https://api.github.com/repos/mozilla/olympia/issues/comments{/number}","contents_url":"https://api.github.com/repos/mozilla/olympia/contents/{+path}","compare_url":"https://api.github.com/repos/mozilla/olympia/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mozilla/olympia/merges","archive_url":"https://api.github.com/repos/mozilla/olympia/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mozilla/olympia/downloads","issues_url":"https://api.github.com/repos/mozilla/olympia/issues{/number}","pulls_url":"https://api.github.com/repos/mozilla/olympia/pulls{/number}","milestones_url":"https://api.github.com/repos/mozilla/olympia/milestones{/number}","notifications_url":"https://api.github.com/repos/mozilla/olympia/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mozilla/olympia/labels{/name}","releases_url":"https://api.github.com/repos/mozilla/olympia/releases{/id}","created_at":"2014-01-31T18:44:15Z","updated_at":"2015-06-15T19:33:47Z","pushed_at":"2015-07-24T07:57:47Z","git_url":"git://github.com/mozilla/olympia.git","ssh_url":"git@github.com:mozilla/olympia.git","clone_url":"https://github.com/mozilla/olympia.git","svn_url":"https://github.com/mozilla/olympia","homepage":"https://addons.mozilla.org/","size":442363,"stargazers_count":35,"watchers_count":35,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":88,"mirror_url":null,"open_issues_count":12,"forks":88,"open_issues":12,"watchers":35,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/mozilla/olympia/pulls/613"},"html":{"href":"https://github.com/mozilla/olympia/pull/613"},"issue":{"href":"https://api.github.com/repos/mozilla/olympia/issues/613"},"comments":{"href":"https://api.github.com/repos/mozilla/olympia/issues/613/comments"},"review_comments":{"href":"https://api.github.com/repos/mozilla/olympia/pulls/613/comments"},"review_comment":{"href":"https://api.github.com/repos/mozilla/olympia/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/mozilla/olympia/pulls/613/commits"},"statuses":{"href":"https://api.github.com/repos/mozilla/olympia/statuses/4cb82e39635e5509862fa16c5e3de9c9efcc9aaf"}}}}