{"action":"created","comment":{"url":"https://api.github.com/repos/socotra/dynomite/pulls/comments/45815848","id":45815848,"diff_hunk":"@@ -1 +1,551 @@\n-from dynomite import Table\n+import json\n+import boto.datapipeline\n+import boto.s3\n+import boto.s3.connection\n+import boto.iam\n+import boto.dynamodb\n+import sys\n+import yaml\n+from string import Template\n+from retrying import retry\n+import time\n+import os\n+from multiprocessing import current_process\n+import logging\n+import uuid\n+\n+\n+def set_logging(level='info'):\n+    logger = logging.getLogger('dynomite')\n+    if not len(logger.handlers):\n+        if level == 'info':\n+            logger.setLevel(logging.INFO)\n+        elif level == 'debug':\n+            logger.setLevel(logging.DEBUG)\n+\n+        ch = logging.StreamHandler()\n+        formatter = logging.Formatter(\n+            '%(asctime)s - %(name)s - %(levelname)s - %(message)s'\n+        )\n+        ch.setFormatter(formatter)\n+        logger.addHandler(ch)\n+\n+    return logger\n+\n+\n+class BackupNotCompleteError(Exception):\n+    def __init__(self, msg):\n+        self.msg = msg\n+\n+\n+class BackupComponentCreationError(Exception):\n+    def __init__(self, msg):\n+        self.msg = msg\n+\n+\n+def dp_connect_to_region(region, access_key_id, secret_access_key_id):\n+    return boto.datapipeline.connect_to_region(\n+        region,\n+        aws_access_key_id=access_key_id,\n+        aws_secret_access_key=secret_access_key_id\n+    )\n+\n+\n+def dynamodb_connect_to_region(region, access_key_id, secret_access_key_id):\n+    return boto.dynamodb.connect_to_region(\n+        region,\n+        aws_access_key_id=access_key_id,\n+        aws_secret_access_key=secret_access_key_id\n+    )\n+\n+\n+def s3_connect_to_region(region, access_key_id, secret_access_key_id):\n+    return boto.s3.connect_to_region(\n+        region,\n+        aws_access_key_id=access_key_id,\n+        aws_secret_access_key=secret_access_key_id\n+    )\n+\n+\n+def get_tables_from_stack_definition(stack_definition):\n+    tables = []\n+    with open(stack_definition, 'r') as f:\n+        config = yaml.load(f.read())\n+\n+    for table in config['aws']['dynamodb']:\n+        try:\n+            if config['aws']['dynamodb'][table]['delete']:\n+                pass\n+        except KeyError:\n+            tables.append(table)\n+\n+    return tables\n+\n+\n+def __create_pipeline_definition__(region, table, action,\n+                                   bucket_name, backup=None):\n+    path = os.path.dirname(os.path.realpath(__file__))\n+    with open(\n+        '%s/templates/%s_pipeline_def.json' % (path, action), 'r'\n+    ) as f:\n+        pipeline_definition = f.read()\n+\n+    d = dict(region=region,\n+             table=table,\n+             bucket_name=bucket_name,\n+             backup=backup)\n+\n+    s = Template(pipeline_definition)\n+\n+    return json.loads(s.safe_substitute(d))\n+\n+\n+def worker(work_queue, done_queue):\n+    for pipeline_options in iter(work_queue.get, 'STOP'):\n+        status = create_pipeline(pipeline_options)\n+        done_queue.put(\"%s - %s\" % (current_process().name, status))\n+\n+    return True\n+\n+\n+def create_pipeline((dp_conn, dynamodb_conn, s3_conn_dest, s3_conn_src,\n+                     region, table, action, read_pt_increase,\n+                     write_pt_increase, date, backup_bucket, debug)):\n+    logger = set_logging(level=debug)\n+    backup = None\n+\n+    logger.info('Executing %s of table %s' % (action, table))\n+    if action == 'restore':\n+        if __check_pt_decreases__:\n+            original_pt = __get_pt__(dynamodb_conn, table)\n+            increased_pt = __increase_pt__(\n+                dynamodb_conn,\n+                table,\n+                original_pt,\n+                debug,\n+                write_increase_percentage=write_pt_increase\n+            )\n+        else:\n+            logger.info(\n+                'Not increasing read PT due to too '\n+                'many decreases on table: %s' % (table)\n+            )\n+        backup = __get_backup__(\n+            s3_conn_src, region, backup_bucket, table, date, debug\n+        )\n+        tmp_bucket_name = __copy_backup_from_source__(\n+            s3_conn_dest, region, table, backup, backup_bucket, debug\n+        )\n+\n+        backup_bucket = tmp_bucket_name\n+\n+        if not backup:\n+            logger.info('No backup exists on %s for table %s' % (date, table))\n+            return True\n+\n+    elif action == 'backup':\n+        if __check_pt_decreases__:\n+            original_pt = __get_pt__(dynamodb_conn, table)\n+            increased_pt = __increase_pt__(\n+                dynamodb_conn,\n+                table,\n+                original_pt,\n+                debug,\n+                read_increase_percentage=read_pt_increase\n+            )\n+        else:\n+            logger.info(\n+                'Not increasing read PT due to too '\n+                'many decreases on table: %s' % (table)\n+            )\n+        try:\n+            s3_conn_src.create_bucket(backup_bucket, location='EU')\n+        except boto.exception.S3CreateError:\n+            logger.debug('Backup bucket creation '\n+                         'failed, %s exists' % backup_bucket)\n+\n+    pipeline_definition = __create_pipeline_definition__(region,\n+                                                         table,\n+                                                         action,\n+                                                         backup_bucket,\n+                                                         backup)\n+\n+    logger.debug(\n+        'Pipeline definition for action:%s and table:%s' % (action, table)\n+    )\n+    logger.debug(pipeline_definition)\n+\n+    pipelines = dp_conn.list_pipelines()['pipelineIdList']\n+    for pipeline in pipelines:\n+        if pipeline['name'] == '%s-%s' % (table, action):\n+            logger.info('Pipeline with name %s-%s exists' % (table, action))\n+            dp_conn.delete_pipeline(pipeline['id'])\n+            logger.info('Pipeline with name %s-%s deleted' % (table, action))\n+\n+    response = dp_conn.create_pipeline('%s-%s' % (table, action), table, None)\n+    pipeline_id = response['pipelineId']\n+    dp_conn.put_pipeline_definition(pipeline_definition,\n+                                    response['pipelineId'])\n+    try:\n+        dp_conn.activate_pipeline(pipeline_id)\n+    except:\n+        logger.error(\n+            'Unable to create pipline %s-%s due to errors '\n+            'in the definition' % (table, action)\n+        )\n+        return True\n+\n+    logger.info('Created pipeline for table: %s' % table)\n+\n+    __check_status__(dp_conn, pipeline_id, table, action, debug)\n+\n+    logger.info('Deleting pipeline for table: %s' % table)\n+    dp_conn.delete_pipeline(pipeline_id)\n+\n+    __restore_pt__(dynamodb_conn, table, original_pt, increased_pt, debug)\n+\n+    if action == 'restore':\n+        logger.info('Deleting temporary bucket %s' % tmp_bucket_name)\n+        b = boto.s3.bucket.Bucket(connection=s3_conn_dest,\n+                                  name=tmp_bucket_name)\n+        for k in b:\n+            k.delete()\n+        b.delete()\n+\n+    return True\n+\n+\n+def __copy_backup_from_source__(s3_conn, region, table,\n+                                backup, src_bucket_name, debug):\n+    logger = set_logging(level=debug)\n+\n+    tmp_bucket_name = str(uuid.uuid4())\n+\n+    dst_bucket = s3_conn.create_bucket(tmp_bucket_name, location='EU')","path":"dynomite/__init__.py","position":225,"original_position":225,"commit_id":"533304e819004bd89f22fa8b308027d434ad4b80","original_commit_id":"533304e819004bd89f22fa8b308027d434ad4b80","user":{"login":"cantenesse","id":786966,"avatar_url":"https://avatars.githubusercontent.com/u/786966?v=3","gravatar_id":"","url":"https://api.github.com/users/cantenesse","html_url":"https://github.com/cantenesse","followers_url":"https://api.github.com/users/cantenesse/followers","following_url":"https://api.github.com/users/cantenesse/following{/other_user}","gists_url":"https://api.github.com/users/cantenesse/gists{/gist_id}","starred_url":"https://api.github.com/users/cantenesse/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cantenesse/subscriptions","organizations_url":"https://api.github.com/users/cantenesse/orgs","repos_url":"https://api.github.com/users/cantenesse/repos","events_url":"https://api.github.com/users/cantenesse/events{/privacy}","received_events_url":"https://api.github.com/users/cantenesse/received_events","type":"User","site_admin":false},"body":"EU is hardcoded for now.  Probably just need an enum (maybe via dict?) to map user provided region to the expected string for the AWS interface.","created_at":"2015-11-25T00:32:29Z","updated_at":"2015-11-25T00:32:29Z","html_url":"https://github.com/socotra/dynomite/pull/20#discussion_r45815848","pull_request_url":"https://api.github.com/repos/socotra/dynomite/pulls/20","_links":{"self":{"href":"https://api.github.com/repos/socotra/dynomite/pulls/comments/45815848"},"html":{"href":"https://github.com/socotra/dynomite/pull/20#discussion_r45815848"},"pull_request":{"href":"https://api.github.com/repos/socotra/dynomite/pulls/20"}}},"pull_request":{"url":"https://api.github.com/repos/socotra/dynomite/pulls/20","id":51727551,"html_url":"https://github.com/socotra/dynomite/pull/20","diff_url":"https://github.com/socotra/dynomite/pull/20.diff","patch_url":"https://github.com/socotra/dynomite/pull/20.patch","issue_url":"https://api.github.com/repos/socotra/dynomite/issues/20","number":20,"state":"open","locked":false,"title":"Uses DP/EMR instead of table scans","user":{"login":"cantenesse","id":786966,"avatar_url":"https://avatars.githubusercontent.com/u/786966?v=3","gravatar_id":"","url":"https://api.github.com/users/cantenesse","html_url":"https://github.com/cantenesse","followers_url":"https://api.github.com/users/cantenesse/followers","following_url":"https://api.github.com/users/cantenesse/following{/other_user}","gists_url":"https://api.github.com/users/cantenesse/gists{/gist_id}","starred_url":"https://api.github.com/users/cantenesse/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cantenesse/subscriptions","organizations_url":"https://api.github.com/users/cantenesse/orgs","repos_url":"https://api.github.com/users/cantenesse/repos","events_url":"https://api.github.com/users/cantenesse/events{/privacy}","received_events_url":"https://api.github.com/users/cantenesse/received_events","type":"User","site_admin":false},"body":"","created_at":"2015-11-24T22:36:49Z","updated_at":"2015-11-25T00:32:29Z","closed_at":null,"merged_at":null,"merge_commit_sha":"6abb968ba142bbf71c4444765e9c891591ac8137","assignee":{"login":"kevinlr5","id":3997211,"avatar_url":"https://avatars.githubusercontent.com/u/3997211?v=3","gravatar_id":"","url":"https://api.github.com/users/kevinlr5","html_url":"https://github.com/kevinlr5","followers_url":"https://api.github.com/users/kevinlr5/followers","following_url":"https://api.github.com/users/kevinlr5/following{/other_user}","gists_url":"https://api.github.com/users/kevinlr5/gists{/gist_id}","starred_url":"https://api.github.com/users/kevinlr5/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kevinlr5/subscriptions","organizations_url":"https://api.github.com/users/kevinlr5/orgs","repos_url":"https://api.github.com/users/kevinlr5/repos","events_url":"https://api.github.com/users/kevinlr5/events{/privacy}","received_events_url":"https://api.github.com/users/kevinlr5/received_events","type":"User","site_admin":false},"milestone":null,"commits_url":"https://api.github.com/repos/socotra/dynomite/pulls/20/commits","review_comments_url":"https://api.github.com/repos/socotra/dynomite/pulls/20/comments","review_comment_url":"https://api.github.com/repos/socotra/dynomite/pulls/comments{/number}","comments_url":"https://api.github.com/repos/socotra/dynomite/issues/20/comments","statuses_url":"https://api.github.com/repos/socotra/dynomite/statuses/533304e819004bd89f22fa8b308027d434ad4b80","head":{"label":"socotra:eng-2582","ref":"eng-2582","sha":"533304e819004bd89f22fa8b308027d434ad4b80","user":{"login":"socotra","id":8236842,"avatar_url":"https://avatars.githubusercontent.com/u/8236842?v=3","gravatar_id":"","url":"https://api.github.com/users/socotra","html_url":"https://github.com/socotra","followers_url":"https://api.github.com/users/socotra/followers","following_url":"https://api.github.com/users/socotra/following{/other_user}","gists_url":"https://api.github.com/users/socotra/gists{/gist_id}","starred_url":"https://api.github.com/users/socotra/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/socotra/subscriptions","organizations_url":"https://api.github.com/users/socotra/orgs","repos_url":"https://api.github.com/users/socotra/repos","events_url":"https://api.github.com/users/socotra/events{/privacy}","received_events_url":"https://api.github.com/users/socotra/received_events","type":"Organization","site_admin":false},"repo":{"id":31744570,"name":"dynomite","full_name":"socotra/dynomite","owner":{"login":"socotra","id":8236842,"avatar_url":"https://avatars.githubusercontent.com/u/8236842?v=3","gravatar_id":"","url":"https://api.github.com/users/socotra","html_url":"https://github.com/socotra","followers_url":"https://api.github.com/users/socotra/followers","following_url":"https://api.github.com/users/socotra/following{/other_user}","gists_url":"https://api.github.com/users/socotra/gists{/gist_id}","starred_url":"https://api.github.com/users/socotra/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/socotra/subscriptions","organizations_url":"https://api.github.com/users/socotra/orgs","repos_url":"https://api.github.com/users/socotra/repos","events_url":"https://api.github.com/users/socotra/events{/privacy}","received_events_url":"https://api.github.com/users/socotra/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/socotra/dynomite","description":"","fork":false,"url":"https://api.github.com/repos/socotra/dynomite","forks_url":"https://api.github.com/repos/socotra/dynomite/forks","keys_url":"https://api.github.com/repos/socotra/dynomite/keys{/key_id}","collaborators_url":"https://api.github.com/repos/socotra/dynomite/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/socotra/dynomite/teams","hooks_url":"https://api.github.com/repos/socotra/dynomite/hooks","issue_events_url":"https://api.github.com/repos/socotra/dynomite/issues/events{/number}","events_url":"https://api.github.com/repos/socotra/dynomite/events","assignees_url":"https://api.github.com/repos/socotra/dynomite/assignees{/user}","branches_url":"https://api.github.com/repos/socotra/dynomite/branches{/branch}","tags_url":"https://api.github.com/repos/socotra/dynomite/tags","blobs_url":"https://api.github.com/repos/socotra/dynomite/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/socotra/dynomite/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/socotra/dynomite/git/refs{/sha}","trees_url":"https://api.github.com/repos/socotra/dynomite/git/trees{/sha}","statuses_url":"https://api.github.com/repos/socotra/dynomite/statuses/{sha}","languages_url":"https://api.github.com/repos/socotra/dynomite/languages","stargazers_url":"https://api.github.com/repos/socotra/dynomite/stargazers","contributors_url":"https://api.github.com/repos/socotra/dynomite/contributors","subscribers_url":"https://api.github.com/repos/socotra/dynomite/subscribers","subscription_url":"https://api.github.com/repos/socotra/dynomite/subscription","commits_url":"https://api.github.com/repos/socotra/dynomite/commits{/sha}","git_commits_url":"https://api.github.com/repos/socotra/dynomite/git/commits{/sha}","comments_url":"https://api.github.com/repos/socotra/dynomite/comments{/number}","issue_comment_url":"https://api.github.com/repos/socotra/dynomite/issues/comments{/number}","contents_url":"https://api.github.com/repos/socotra/dynomite/contents/{+path}","compare_url":"https://api.github.com/repos/socotra/dynomite/compare/{base}...{head}","merges_url":"https://api.github.com/repos/socotra/dynomite/merges","archive_url":"https://api.github.com/repos/socotra/dynomite/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/socotra/dynomite/downloads","issues_url":"https://api.github.com/repos/socotra/dynomite/issues{/number}","pulls_url":"https://api.github.com/repos/socotra/dynomite/pulls{/number}","milestones_url":"https://api.github.com/repos/socotra/dynomite/milestones{/number}","notifications_url":"https://api.github.com/repos/socotra/dynomite/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/socotra/dynomite/labels{/name}","releases_url":"https://api.github.com/repos/socotra/dynomite/releases{/id}","created_at":"2015-03-06T00:35:53Z","updated_at":"2015-07-13T20:55:34Z","pushed_at":"2015-11-24T22:53:52Z","git_url":"git://github.com/socotra/dynomite.git","ssh_url":"git@github.com:socotra/dynomite.git","clone_url":"https://github.com/socotra/dynomite.git","svn_url":"https://github.com/socotra/dynomite","homepage":null,"size":60,"stargazers_count":1,"watchers_count":1,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":1,"default_branch":"master"}},"base":{"label":"socotra:develop","ref":"develop","sha":"738b0949159477b1a43b0573780d11ca3197f0c8","user":{"login":"socotra","id":8236842,"avatar_url":"https://avatars.githubusercontent.com/u/8236842?v=3","gravatar_id":"","url":"https://api.github.com/users/socotra","html_url":"https://github.com/socotra","followers_url":"https://api.github.com/users/socotra/followers","following_url":"https://api.github.com/users/socotra/following{/other_user}","gists_url":"https://api.github.com/users/socotra/gists{/gist_id}","starred_url":"https://api.github.com/users/socotra/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/socotra/subscriptions","organizations_url":"https://api.github.com/users/socotra/orgs","repos_url":"https://api.github.com/users/socotra/repos","events_url":"https://api.github.com/users/socotra/events{/privacy}","received_events_url":"https://api.github.com/users/socotra/received_events","type":"Organization","site_admin":false},"repo":{"id":31744570,"name":"dynomite","full_name":"socotra/dynomite","owner":{"login":"socotra","id":8236842,"avatar_url":"https://avatars.githubusercontent.com/u/8236842?v=3","gravatar_id":"","url":"https://api.github.com/users/socotra","html_url":"https://github.com/socotra","followers_url":"https://api.github.com/users/socotra/followers","following_url":"https://api.github.com/users/socotra/following{/other_user}","gists_url":"https://api.github.com/users/socotra/gists{/gist_id}","starred_url":"https://api.github.com/users/socotra/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/socotra/subscriptions","organizations_url":"https://api.github.com/users/socotra/orgs","repos_url":"https://api.github.com/users/socotra/repos","events_url":"https://api.github.com/users/socotra/events{/privacy}","received_events_url":"https://api.github.com/users/socotra/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/socotra/dynomite","description":"","fork":false,"url":"https://api.github.com/repos/socotra/dynomite","forks_url":"https://api.github.com/repos/socotra/dynomite/forks","keys_url":"https://api.github.com/repos/socotra/dynomite/keys{/key_id}","collaborators_url":"https://api.github.com/repos/socotra/dynomite/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/socotra/dynomite/teams","hooks_url":"https://api.github.com/repos/socotra/dynomite/hooks","issue_events_url":"https://api.github.com/repos/socotra/dynomite/issues/events{/number}","events_url":"https://api.github.com/repos/socotra/dynomite/events","assignees_url":"https://api.github.com/repos/socotra/dynomite/assignees{/user}","branches_url":"https://api.github.com/repos/socotra/dynomite/branches{/branch}","tags_url":"https://api.github.com/repos/socotra/dynomite/tags","blobs_url":"https://api.github.com/repos/socotra/dynomite/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/socotra/dynomite/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/socotra/dynomite/git/refs{/sha}","trees_url":"https://api.github.com/repos/socotra/dynomite/git/trees{/sha}","statuses_url":"https://api.github.com/repos/socotra/dynomite/statuses/{sha}","languages_url":"https://api.github.com/repos/socotra/dynomite/languages","stargazers_url":"https://api.github.com/repos/socotra/dynomite/stargazers","contributors_url":"https://api.github.com/repos/socotra/dynomite/contributors","subscribers_url":"https://api.github.com/repos/socotra/dynomite/subscribers","subscription_url":"https://api.github.com/repos/socotra/dynomite/subscription","commits_url":"https://api.github.com/repos/socotra/dynomite/commits{/sha}","git_commits_url":"https://api.github.com/repos/socotra/dynomite/git/commits{/sha}","comments_url":"https://api.github.com/repos/socotra/dynomite/comments{/number}","issue_comment_url":"https://api.github.com/repos/socotra/dynomite/issues/comments{/number}","contents_url":"https://api.github.com/repos/socotra/dynomite/contents/{+path}","compare_url":"https://api.github.com/repos/socotra/dynomite/compare/{base}...{head}","merges_url":"https://api.github.com/repos/socotra/dynomite/merges","archive_url":"https://api.github.com/repos/socotra/dynomite/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/socotra/dynomite/downloads","issues_url":"https://api.github.com/repos/socotra/dynomite/issues{/number}","pulls_url":"https://api.github.com/repos/socotra/dynomite/pulls{/number}","milestones_url":"https://api.github.com/repos/socotra/dynomite/milestones{/number}","notifications_url":"https://api.github.com/repos/socotra/dynomite/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/socotra/dynomite/labels{/name}","releases_url":"https://api.github.com/repos/socotra/dynomite/releases{/id}","created_at":"2015-03-06T00:35:53Z","updated_at":"2015-07-13T20:55:34Z","pushed_at":"2015-11-24T22:53:52Z","git_url":"git://github.com/socotra/dynomite.git","ssh_url":"git@github.com:socotra/dynomite.git","clone_url":"https://github.com/socotra/dynomite.git","svn_url":"https://github.com/socotra/dynomite","homepage":null,"size":60,"stargazers_count":1,"watchers_count":1,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":1,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/socotra/dynomite/pulls/20"},"html":{"href":"https://github.com/socotra/dynomite/pull/20"},"issue":{"href":"https://api.github.com/repos/socotra/dynomite/issues/20"},"comments":{"href":"https://api.github.com/repos/socotra/dynomite/issues/20/comments"},"review_comments":{"href":"https://api.github.com/repos/socotra/dynomite/pulls/20/comments"},"review_comment":{"href":"https://api.github.com/repos/socotra/dynomite/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/socotra/dynomite/pulls/20/commits"},"statuses":{"href":"https://api.github.com/repos/socotra/dynomite/statuses/533304e819004bd89f22fa8b308027d434ad4b80"}}}}