{"action":"created","comment":{"url":"https://api.github.com/repos/mozilla/build-relengapi/pulls/comments/32570080","id":32570080,"diff_hunk":"@@ -0,0 +1,111 @@\n+# This Source Code Form is subject to the terms of the Mozilla Public\n+# License, v. 2.0. If a copy of the MPL was not distributed with this\n+# file, You can obtain one at http://mozilla.org/MPL/2.0/.\n+\n+import logging\n+import os\n+\n+from flask import Blueprint\n+from flask import current_app\n+from flask import redirect\n+from flask import url_for\n+from relengapi.blueprints.archiver.tasks import create_and_upload_archive\n+from relengapi.blueprints.archiver.types import MozharnessArchiveTask\n+from relengapi.lib import api\n+from werkzeug.exceptions import NotFound\n+\n+bp = Blueprint('archiver', __name__)\n+log = logging.getLogger(__name__)\n+\n+GET_EXPIRES_IN = 300\n+\n+\n+@bp.route('/status/<task_id>')\n+@api.apimethod(MozharnessArchiveTask, unicode)\n+def task_status(task_id):\n+    \"\"\"\n+    Check and return the current state of the create_and_upload_archive celery task with task id\n+    of <task_id>.\n+\n+    If the task is unknown, state will be PENDING. Once the task starts it will be updated to\n+    STARTED and finally, if it completes, it will be either SUCCESS (no exceptions), or FAILURE.\n+\n+    See update_state() within create_and_upload_archive and\n+    http://celery.readthedocs.org/en/latest/reference/celery.states.html for more details.\n+\n+    If state is SUCCESS, it is safe to check response['s3_urls'] for the archives submitted to s3\n+    \"\"\"\n+    task = create_and_upload_archive.AsyncResult(task_id)\n+    task_info = task.info or {}\n+    response = {\n+        'state': task.state,\n+        'src_url': task_info.get('src_url', ''),\n+        's3_urls': task_info.get('s3_urls', {})\n+    }\n+    if task.state != 'FAILURE':\n+        response['status'] = task_info.get('status', 'no status available at this point.')\n+    else:\n+        # something went wrong\n+        response['status'] = str(task.info)  # this is the exception raised\n+\n+    return MozharnessArchiveTask(**response)\n+\n+\n+@bp.route('/mozharness/<rev>')\n+@api.apimethod(None, unicode, unicode, unicode, unicode, status_code=302)\n+def get_mozharness_archive(rev, repo=\"mozilla-central\", region='us-west-2', suffix='tar.gz'):\n+    cfg = current_app.config['SUBREPO_MOZHARNESS_CFG']\n+    return get_archive_from_repo(cfg, rev, repo, region, suffix)\n+\n+\n+def get_archive_from_repo(cfg, rev, repo, region, suffix):\n+    \"\"\"\n+    A generic getter for retrieving an s3 location of an archive where the archive is based off a\n+    given repo name, revision, and possibly sub-dir.\n+\n+    sub-dir: hg.mozilla.org supports archives of sub directories within a repository. This\n+    flexibility allows for creating archives of only a portion of what would normally be an entire\n+    repo archive.\n+\n+    logic flow:\n+     If their is already a key based on given args, a re-direct link is given for the\n+    s3 location. If the key does not exist, download the archive from src url, upload it to and\n+    return all s3 url locations.\n+\n+     When the key does not exist, the remaining work will be assigned to a celery background task\n+    with a url location returned immediately for obtaining task state updates.\n+    \"\"\"\n+    bucket_region = None\n+    bucket_name = None\n+    for bucket in cfg['S3_BUCKETS']:\n+        if region in bucket['REGION']:\n+            bucket_region = bucket['REGION']\n+            bucket_name = bucket['NAME']\n+\n+    # sanity check\n+    if not bucket_name or not bucket_region:\n+        valid_regions = str([bucket['REGION'] for bucket in cfg['S3_BUCKETS']])\n+        log.warning('Unsupported region given: \"{}\" Valid Regions \"{}\"'.format(region,\n+                                                                               valid_regions))\n+        raise NotFound\n+\n+    s3 = current_app.aws.connect_to('s3', bucket_region)\n+    bucket = s3.get_bucket(bucket_name)\n+    key = '{repo}-{rev}.{suffix}'.format(repo=os.path.basename(repo), rev=rev, suffix=suffix)","path":"relengapi/blueprints/archiver/__init__.py","position":94,"original_position":94,"commit_id":"a85df9c40353719fa6e1bb9c656e7175abd69c01","original_commit_id":"a85df9c40353719fa6e1bb9c656e7175abd69c01","user":{"login":"djmitche","id":28673,"avatar_url":"https://avatars.githubusercontent.com/u/28673?v=3","gravatar_id":"","url":"https://api.github.com/users/djmitche","html_url":"https://github.com/djmitche","followers_url":"https://api.github.com/users/djmitche/followers","following_url":"https://api.github.com/users/djmitche/following{/other_user}","gists_url":"https://api.github.com/users/djmitche/gists{/gist_id}","starred_url":"https://api.github.com/users/djmitche/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/djmitche/subscriptions","organizations_url":"https://api.github.com/users/djmitche/orgs","repos_url":"https://api.github.com/users/djmitche/repos","events_url":"https://api.github.com/users/djmitche/events{/privacy}","received_events_url":"https://api.github.com/users/djmitche/received_events","type":"User","site_admin":false},"body":"This could lead to collisions if we have repos with the same basename.  Might as well use the whole repo string.","created_at":"2015-06-16T20:59:56Z","updated_at":"2015-06-16T20:59:56Z","html_url":"https://github.com/mozilla/build-relengapi/pull/279#discussion_r32570080","pull_request_url":"https://api.github.com/repos/mozilla/build-relengapi/pulls/279","_links":{"self":{"href":"https://api.github.com/repos/mozilla/build-relengapi/pulls/comments/32570080"},"html":{"href":"https://github.com/mozilla/build-relengapi/pull/279#discussion_r32570080"},"pull_request":{"href":"https://api.github.com/repos/mozilla/build-relengapi/pulls/279"}}},"pull_request":{"url":"https://api.github.com/repos/mozilla/build-relengapi/pulls/279","id":37309028,"html_url":"https://github.com/mozilla/build-relengapi/pull/279","diff_url":"https://github.com/mozilla/build-relengapi/pull/279.diff","patch_url":"https://github.com/mozilla/build-relengapi/pull/279.patch","issue_url":"https://api.github.com/repos/mozilla/build-relengapi/issues/279","number":279,"state":"open","locked":false,"title":"Bug 1154795 - add an endpoint to relengapi that creates and returns a Mozharness archive via S3","user":{"login":"lundjordan","id":1648433,"avatar_url":"https://avatars.githubusercontent.com/u/1648433?v=3","gravatar_id":"","url":"https://api.github.com/users/lundjordan","html_url":"https://github.com/lundjordan","followers_url":"https://api.github.com/users/lundjordan/followers","following_url":"https://api.github.com/users/lundjordan/following{/other_user}","gists_url":"https://api.github.com/users/lundjordan/gists{/gist_id}","starred_url":"https://api.github.com/users/lundjordan/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/lundjordan/subscriptions","organizations_url":"https://api.github.com/users/lundjordan/orgs","repos_url":"https://api.github.com/users/lundjordan/repos","events_url":"https://api.github.com/users/lundjordan/events{/privacy}","received_events_url":"https://api.github.com/users/lundjordan/received_events","type":"User","site_admin":false},"body":"this consolidates my relengapi repo into the now unified build-relengapi and addresses feedback comments regarding:\r\n- hardcoded repo and subdir: now supports any hg repo, branch, and subdir configuration\r\n- hardcoded regions: regions are in the cfg\r\n- blueprints no longer copy skeleton repo style of blueprints/name_of_blueprint.py and instead uses blueprints/name_of_blueprint/{__init__.py, etc}\r\n- my settings.py cfg uses https not http\r\n- I ended up downloading the src archive to disk but in a tempfile\r\n\r\nfrom feedback that is not implemented:\r\n- I did not end up using celery subtasks as I felt it may be overkill for the situation\r\n- I also did not use celery's retry because in the event of a FAILURE, the current logic has it that if the s3 key being requested does not exist, the task to download and upload it will just be re-created again anyway. The need to support retry didn't seem overly important due to this.\r\n ","created_at":"2015-06-09T22:53:31Z","updated_at":"2015-06-16T20:59:56Z","closed_at":null,"merged_at":null,"merge_commit_sha":"ccf43c56bc8f665e70b3d2ae29496bd705cfd2ac","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/mozilla/build-relengapi/pulls/279/commits","review_comments_url":"https://api.github.com/repos/mozilla/build-relengapi/pulls/279/comments","review_comment_url":"https://api.github.com/repos/mozilla/build-relengapi/pulls/comments{/number}","comments_url":"https://api.github.com/repos/mozilla/build-relengapi/issues/279/comments","statuses_url":"https://api.github.com/repos/mozilla/build-relengapi/statuses/a85df9c40353719fa6e1bb9c656e7175abd69c01","head":{"label":"lundjordan:archiver","ref":"archiver","sha":"a85df9c40353719fa6e1bb9c656e7175abd69c01","user":{"login":"lundjordan","id":1648433,"avatar_url":"https://avatars.githubusercontent.com/u/1648433?v=3","gravatar_id":"","url":"https://api.github.com/users/lundjordan","html_url":"https://github.com/lundjordan","followers_url":"https://api.github.com/users/lundjordan/followers","following_url":"https://api.github.com/users/lundjordan/following{/other_user}","gists_url":"https://api.github.com/users/lundjordan/gists{/gist_id}","starred_url":"https://api.github.com/users/lundjordan/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/lundjordan/subscriptions","organizations_url":"https://api.github.com/users/lundjordan/orgs","repos_url":"https://api.github.com/users/lundjordan/repos","events_url":"https://api.github.com/users/lundjordan/events{/privacy}","received_events_url":"https://api.github.com/users/lundjordan/received_events","type":"User","site_admin":false},"repo":{"id":19584971,"name":"build-relengapi","full_name":"lundjordan/build-relengapi","owner":{"login":"lundjordan","id":1648433,"avatar_url":"https://avatars.githubusercontent.com/u/1648433?v=3","gravatar_id":"","url":"https://api.github.com/users/lundjordan","html_url":"https://github.com/lundjordan","followers_url":"https://api.github.com/users/lundjordan/followers","following_url":"https://api.github.com/users/lundjordan/following{/other_user}","gists_url":"https://api.github.com/users/lundjordan/gists{/gist_id}","starred_url":"https://api.github.com/users/lundjordan/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/lundjordan/subscriptions","organizations_url":"https://api.github.com/users/lundjordan/orgs","repos_url":"https://api.github.com/users/lundjordan/repos","events_url":"https://api.github.com/users/lundjordan/events{/privacy}","received_events_url":"https://api.github.com/users/lundjordan/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/lundjordan/build-relengapi","description":"Your Interface to Release Engineering Automation -","fork":true,"url":"https://api.github.com/repos/lundjordan/build-relengapi","forks_url":"https://api.github.com/repos/lundjordan/build-relengapi/forks","keys_url":"https://api.github.com/repos/lundjordan/build-relengapi/keys{/key_id}","collaborators_url":"https://api.github.com/repos/lundjordan/build-relengapi/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/lundjordan/build-relengapi/teams","hooks_url":"https://api.github.com/repos/lundjordan/build-relengapi/hooks","issue_events_url":"https://api.github.com/repos/lundjordan/build-relengapi/issues/events{/number}","events_url":"https://api.github.com/repos/lundjordan/build-relengapi/events","assignees_url":"https://api.github.com/repos/lundjordan/build-relengapi/assignees{/user}","branches_url":"https://api.github.com/repos/lundjordan/build-relengapi/branches{/branch}","tags_url":"https://api.github.com/repos/lundjordan/build-relengapi/tags","blobs_url":"https://api.github.com/repos/lundjordan/build-relengapi/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/lundjordan/build-relengapi/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/lundjordan/build-relengapi/git/refs{/sha}","trees_url":"https://api.github.com/repos/lundjordan/build-relengapi/git/trees{/sha}","statuses_url":"https://api.github.com/repos/lundjordan/build-relengapi/statuses/{sha}","languages_url":"https://api.github.com/repos/lundjordan/build-relengapi/languages","stargazers_url":"https://api.github.com/repos/lundjordan/build-relengapi/stargazers","contributors_url":"https://api.github.com/repos/lundjordan/build-relengapi/contributors","subscribers_url":"https://api.github.com/repos/lundjordan/build-relengapi/subscribers","subscription_url":"https://api.github.com/repos/lundjordan/build-relengapi/subscription","commits_url":"https://api.github.com/repos/lundjordan/build-relengapi/commits{/sha}","git_commits_url":"https://api.github.com/repos/lundjordan/build-relengapi/git/commits{/sha}","comments_url":"https://api.github.com/repos/lundjordan/build-relengapi/comments{/number}","issue_comment_url":"https://api.github.com/repos/lundjordan/build-relengapi/issues/comments{/number}","contents_url":"https://api.github.com/repos/lundjordan/build-relengapi/contents/{+path}","compare_url":"https://api.github.com/repos/lundjordan/build-relengapi/compare/{base}...{head}","merges_url":"https://api.github.com/repos/lundjordan/build-relengapi/merges","archive_url":"https://api.github.com/repos/lundjordan/build-relengapi/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/lundjordan/build-relengapi/downloads","issues_url":"https://api.github.com/repos/lundjordan/build-relengapi/issues{/number}","pulls_url":"https://api.github.com/repos/lundjordan/build-relengapi/pulls{/number}","milestones_url":"https://api.github.com/repos/lundjordan/build-relengapi/milestones{/number}","notifications_url":"https://api.github.com/repos/lundjordan/build-relengapi/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/lundjordan/build-relengapi/labels{/name}","releases_url":"https://api.github.com/repos/lundjordan/build-relengapi/releases{/id}","created_at":"2014-05-08T18:50:34Z","updated_at":"2015-06-09T22:23:06Z","pushed_at":"2015-06-16T07:08:57Z","git_url":"git://github.com/lundjordan/build-relengapi.git","ssh_url":"git@github.com:lundjordan/build-relengapi.git","clone_url":"https://github.com/lundjordan/build-relengapi.git","svn_url":"https://github.com/lundjordan/build-relengapi","homepage":"https://api.pub.build.mozilla.org","size":20302,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"mozilla:master","ref":"master","sha":"cca7b2f2b67773e1f0475e9b321b8240ed6c282c","user":{"login":"mozilla","id":131524,"avatar_url":"https://avatars.githubusercontent.com/u/131524?v=3","gravatar_id":"","url":"https://api.github.com/users/mozilla","html_url":"https://github.com/mozilla","followers_url":"https://api.github.com/users/mozilla/followers","following_url":"https://api.github.com/users/mozilla/following{/other_user}","gists_url":"https://api.github.com/users/mozilla/gists{/gist_id}","starred_url":"https://api.github.com/users/mozilla/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mozilla/subscriptions","organizations_url":"https://api.github.com/users/mozilla/orgs","repos_url":"https://api.github.com/users/mozilla/repos","events_url":"https://api.github.com/users/mozilla/events{/privacy}","received_events_url":"https://api.github.com/users/mozilla/received_events","type":"Organization","site_admin":false},"repo":{"id":18413379,"name":"build-relengapi","full_name":"mozilla/build-relengapi","owner":{"login":"mozilla","id":131524,"avatar_url":"https://avatars.githubusercontent.com/u/131524?v=3","gravatar_id":"","url":"https://api.github.com/users/mozilla","html_url":"https://github.com/mozilla","followers_url":"https://api.github.com/users/mozilla/followers","following_url":"https://api.github.com/users/mozilla/following{/other_user}","gists_url":"https://api.github.com/users/mozilla/gists{/gist_id}","starred_url":"https://api.github.com/users/mozilla/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mozilla/subscriptions","organizations_url":"https://api.github.com/users/mozilla/orgs","repos_url":"https://api.github.com/users/mozilla/repos","events_url":"https://api.github.com/users/mozilla/events{/privacy}","received_events_url":"https://api.github.com/users/mozilla/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/mozilla/build-relengapi","description":"Your Interface to Release Engineering Automation -","fork":false,"url":"https://api.github.com/repos/mozilla/build-relengapi","forks_url":"https://api.github.com/repos/mozilla/build-relengapi/forks","keys_url":"https://api.github.com/repos/mozilla/build-relengapi/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mozilla/build-relengapi/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mozilla/build-relengapi/teams","hooks_url":"https://api.github.com/repos/mozilla/build-relengapi/hooks","issue_events_url":"https://api.github.com/repos/mozilla/build-relengapi/issues/events{/number}","events_url":"https://api.github.com/repos/mozilla/build-relengapi/events","assignees_url":"https://api.github.com/repos/mozilla/build-relengapi/assignees{/user}","branches_url":"https://api.github.com/repos/mozilla/build-relengapi/branches{/branch}","tags_url":"https://api.github.com/repos/mozilla/build-relengapi/tags","blobs_url":"https://api.github.com/repos/mozilla/build-relengapi/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mozilla/build-relengapi/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mozilla/build-relengapi/git/refs{/sha}","trees_url":"https://api.github.com/repos/mozilla/build-relengapi/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mozilla/build-relengapi/statuses/{sha}","languages_url":"https://api.github.com/repos/mozilla/build-relengapi/languages","stargazers_url":"https://api.github.com/repos/mozilla/build-relengapi/stargazers","contributors_url":"https://api.github.com/repos/mozilla/build-relengapi/contributors","subscribers_url":"https://api.github.com/repos/mozilla/build-relengapi/subscribers","subscription_url":"https://api.github.com/repos/mozilla/build-relengapi/subscription","commits_url":"https://api.github.com/repos/mozilla/build-relengapi/commits{/sha}","git_commits_url":"https://api.github.com/repos/mozilla/build-relengapi/git/commits{/sha}","comments_url":"https://api.github.com/repos/mozilla/build-relengapi/comments{/number}","issue_comment_url":"https://api.github.com/repos/mozilla/build-relengapi/issues/comments{/number}","contents_url":"https://api.github.com/repos/mozilla/build-relengapi/contents/{+path}","compare_url":"https://api.github.com/repos/mozilla/build-relengapi/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mozilla/build-relengapi/merges","archive_url":"https://api.github.com/repos/mozilla/build-relengapi/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mozilla/build-relengapi/downloads","issues_url":"https://api.github.com/repos/mozilla/build-relengapi/issues{/number}","pulls_url":"https://api.github.com/repos/mozilla/build-relengapi/pulls{/number}","milestones_url":"https://api.github.com/repos/mozilla/build-relengapi/milestones{/number}","notifications_url":"https://api.github.com/repos/mozilla/build-relengapi/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mozilla/build-relengapi/labels{/name}","releases_url":"https://api.github.com/repos/mozilla/build-relengapi/releases{/id}","created_at":"2014-04-03T18:20:38Z","updated_at":"2015-05-08T19:02:40Z","pushed_at":"2015-06-16T20:13:53Z","git_url":"git://github.com/mozilla/build-relengapi.git","ssh_url":"git@github.com:mozilla/build-relengapi.git","clone_url":"https://github.com/mozilla/build-relengapi.git","svn_url":"https://github.com/mozilla/build-relengapi","homepage":"https://api.pub.build.mozilla.org","size":21389,"stargazers_count":6,"watchers_count":6,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":15,"mirror_url":null,"open_issues_count":79,"forks":15,"open_issues":79,"watchers":6,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/mozilla/build-relengapi/pulls/279"},"html":{"href":"https://github.com/mozilla/build-relengapi/pull/279"},"issue":{"href":"https://api.github.com/repos/mozilla/build-relengapi/issues/279"},"comments":{"href":"https://api.github.com/repos/mozilla/build-relengapi/issues/279/comments"},"review_comments":{"href":"https://api.github.com/repos/mozilla/build-relengapi/pulls/279/comments"},"review_comment":{"href":"https://api.github.com/repos/mozilla/build-relengapi/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/mozilla/build-relengapi/pulls/279/commits"},"statuses":{"href":"https://api.github.com/repos/mozilla/build-relengapi/statuses/a85df9c40353719fa6e1bb9c656e7175abd69c01"}}}}