{"action":"created","comment":{"url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/pulls/comments/29148172","id":29148172,"diff_hunk":"@@ -0,0 +1,74 @@\n+\"\"\"Helper functions to handle AMI creation with packer\"\"\"\n+\n+import boto\n+import os\n+import subprocess\n+\n+import logging\n+import urllib2\n+import csv\n+\n+LOGGER = logging.getLogger('geotrellis_spark')\n+\n+\n+class GTAMIException(Exception):\n+    pass\n+\n+\n+def get_ubuntu_ami(region):\n+    \"\"\"Gets AMI ID for current release in region\"\"\"\n+    response = urllib2.urlopen('http://cloud-images.ubuntu.com/query/trusty/server/released.current.txt').readlines()\n+    fieldnames = ['version', 'version_type', 'release_status', 'date',\n+                  'storage', 'arch', 'region', 'id', 'kernel',\n+                  'unknown_col', 'virtualization_type']\n+    reader = csv.DictReader(response, fieldnames=fieldnames, delimiter='\\t')\n+\n+    def ami_filter(ami):\n+        \"\"\"Helper function to filter AMIs\"\"\"\n+        return (ami['region'] == region and\n+                ami['arch'] == 'amd64' and\n+                ami['storage'] == 'ebs-ssd' and\n+                ami['virtualization_type'] == 'hvm')\n+\n+    amis = [row for row in reader if ami_filter(row)]\n+    if len(amis) == 0:\n+        raise GTAMIException('Did not find any ubuntu AMIs to use')\n+    elif len(amis) > 1:\n+        raise GTAMIException('Found multiple ubuntu AMIs to use, should only be one')\n+    return amis[0]['id']\n+\n+\n+def run_packer(machine_type, aws_profile, region, stack_type):\n+    \"\"\"Function to run packer\n+\n+    Args:\n+      machine_type (str): type of machine to build (e.g. mesos-leader, mesos-follower)\n+      aws_profile (str): aws profile name to use for authentication\n+      stack_type (str): type of stack this machine is for (e.g. accumulo geotrellis cluster)\n+    \"\"\"\n+\n+    # credentials\n+    aws_dir = os.path.expanduser('~/.aws')\n+    boto_config_path = os.path.join(aws_dir, 'config')\n+    aws_creds_path = os.path.join(aws_dir, 'credentials')\n+    boto.config.read([boto_config_path, aws_creds_path])\n+    aws_access_key_id = boto.config.get(aws_profile, 'aws_access_key_id')\n+    aws_secret_access_key = boto.config.get(aws_profile, 'aws_secret_access_key')\n+\n+    aws_ubuntu_ami = get_ubuntu_ami(region)\n+\n+    # use environment variables when running packer because you cannot specify an\n+    # AWS profile in ~/.aws/credentials to use with packer\n+    env = os.environ.copy()\n+    env['AWS_ACCESS_KEY'] = aws_access_key_id\n+    env['AWS_SECRET_ACCESS_KEY'] = aws_secret_access_key\n+    packer_template_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'template.js')\n+    LOGGER.info('Creating %s AMI in %s region', machine_type, region)\n+    packer_command = ['packer', 'build',","path":"deployment/packer/gt_packer.py","position":67,"original_position":67,"commit_id":"aacd69970f42118268f3e3052aa6e8e3e99b520f","original_commit_id":"aacd69970f42118268f3e3052aa6e8e3e99b520f","user":{"login":"hectcastro","id":43639,"avatar_url":"https://avatars.githubusercontent.com/u/43639?v=3","gravatar_id":"","url":"https://api.github.com/users/hectcastro","html_url":"https://github.com/hectcastro","followers_url":"https://api.github.com/users/hectcastro/followers","following_url":"https://api.github.com/users/hectcastro/following{/other_user}","gists_url":"https://api.github.com/users/hectcastro/gists{/gist_id}","starred_url":"https://api.github.com/users/hectcastro/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hectcastro/subscriptions","organizations_url":"https://api.github.com/users/hectcastro/orgs","repos_url":"https://api.github.com/users/hectcastro/repos","events_url":"https://api.github.com/users/hectcastro/events{/privacy}","received_events_url":"https://api.github.com/users/hectcastro/received_events","type":"User","site_admin":false},"body":"Opened https://github.com/geotrellis/geotrellis-ec2-cluster/issues/27 for this.","created_at":"2015-04-27T14:02:34Z","updated_at":"2015-04-27T14:02:34Z","html_url":"https://github.com/geotrellis/geotrellis-ec2-cluster/pull/26#discussion_r29148172","pull_request_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/pulls/26","_links":{"self":{"href":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/pulls/comments/29148172"},"html":{"href":"https://github.com/geotrellis/geotrellis-ec2-cluster/pull/26#discussion_r29148172"},"pull_request":{"href":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/pulls/26"}}},"pull_request":{"url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/pulls/26","id":33948345,"html_url":"https://github.com/geotrellis/geotrellis-ec2-cluster/pull/26","diff_url":"https://github.com/geotrellis/geotrellis-ec2-cluster/pull/26.diff","patch_url":"https://github.com/geotrellis/geotrellis-ec2-cluster/pull/26.patch","issue_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/issues/26","number":26,"state":"open","locked":false,"title":"Make scripts to be region/account agnostic","user":{"login":"notthatbreezy","id":898060,"avatar_url":"https://avatars.githubusercontent.com/u/898060?v=3","gravatar_id":"","url":"https://api.github.com/users/notthatbreezy","html_url":"https://github.com/notthatbreezy","followers_url":"https://api.github.com/users/notthatbreezy/followers","following_url":"https://api.github.com/users/notthatbreezy/following{/other_user}","gists_url":"https://api.github.com/users/notthatbreezy/gists{/gist_id}","starred_url":"https://api.github.com/users/notthatbreezy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/notthatbreezy/subscriptions","organizations_url":"https://api.github.com/users/notthatbreezy/orgs","repos_url":"https://api.github.com/users/notthatbreezy/repos","events_url":"https://api.github.com/users/notthatbreezy/events{/privacy}","received_events_url":"https://api.github.com/users/notthatbreezy/received_events","type":"User","site_admin":false},"body":"This PR refactors the cluster scripts/setup to be more flexible and region/account agnostic for setting up a geotrellis ec2 cluster.\r\n\r\nThe main features now are that the creation of AMIs and launching of stacks are driven by a config file (`geotrellis-cluster.config`) that will make it easy to extend and launch different types of stacks (`accumulo`, `cassandra`, `hdfs`, etc.) going forward.\r\n\r\nEverything is driven by the `gt-stack.py` CLI with subcommands for creating AMIs and launching of stacks. I've used this to launch stacks in a few different regions and haven't noticed any issues.\r\n\r\nA good start to reviewing this would be reading the new deployment README and seeing if anything is confusing or unclear.\r\n\r\nThere are a few things that would be nice-to-haves, but I don't think are super necessary. They wouldn't be hard to do I don't think, but I wanted to get this out there first.\r\n- remove copy-pasted `MajorKirby` code to use the pip installable one that was just added to github\r\n- add additional sub-commands for listing AMIs, listing stacks, deleting stacks, etc.","created_at":"2015-04-23T11:59:22Z","updated_at":"2015-04-27T14:02:34Z","closed_at":null,"merged_at":null,"merge_commit_sha":"37a9ed311fab3234411e1a6a950a3a7b1fae6fcc","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/pulls/26/commits","review_comments_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/pulls/26/comments","review_comment_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/pulls/comments{/number}","comments_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/issues/26/comments","statuses_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/statuses/aacd69970f42118268f3e3052aa6e8e3e99b520f","head":{"label":"notthatbreezy:feature/refactor-deployment-scripts","ref":"feature/refactor-deployment-scripts","sha":"aacd69970f42118268f3e3052aa6e8e3e99b520f","user":{"login":"notthatbreezy","id":898060,"avatar_url":"https://avatars.githubusercontent.com/u/898060?v=3","gravatar_id":"","url":"https://api.github.com/users/notthatbreezy","html_url":"https://github.com/notthatbreezy","followers_url":"https://api.github.com/users/notthatbreezy/followers","following_url":"https://api.github.com/users/notthatbreezy/following{/other_user}","gists_url":"https://api.github.com/users/notthatbreezy/gists{/gist_id}","starred_url":"https://api.github.com/users/notthatbreezy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/notthatbreezy/subscriptions","organizations_url":"https://api.github.com/users/notthatbreezy/orgs","repos_url":"https://api.github.com/users/notthatbreezy/repos","events_url":"https://api.github.com/users/notthatbreezy/events{/privacy}","received_events_url":"https://api.github.com/users/notthatbreezy/received_events","type":"User","site_admin":false},"repo":{"id":28636874,"name":"vagrant-geotrellis-mesos-spark","full_name":"notthatbreezy/vagrant-geotrellis-mesos-spark","owner":{"login":"notthatbreezy","id":898060,"avatar_url":"https://avatars.githubusercontent.com/u/898060?v=3","gravatar_id":"","url":"https://api.github.com/users/notthatbreezy","html_url":"https://github.com/notthatbreezy","followers_url":"https://api.github.com/users/notthatbreezy/followers","following_url":"https://api.github.com/users/notthatbreezy/following{/other_user}","gists_url":"https://api.github.com/users/notthatbreezy/gists{/gist_id}","starred_url":"https://api.github.com/users/notthatbreezy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/notthatbreezy/subscriptions","organizations_url":"https://api.github.com/users/notthatbreezy/orgs","repos_url":"https://api.github.com/users/notthatbreezy/repos","events_url":"https://api.github.com/users/notthatbreezy/events{/privacy}","received_events_url":"https://api.github.com/users/notthatbreezy/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/notthatbreezy/vagrant-geotrellis-mesos-spark","description":"How many buzzwords can I fit into one repository name?","fork":true,"url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark","forks_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/forks","keys_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/teams","hooks_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/hooks","issue_events_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/issues/events{/number}","events_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/events","assignees_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/assignees{/user}","branches_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/branches{/branch}","tags_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/tags","blobs_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/statuses/{sha}","languages_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/languages","stargazers_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/stargazers","contributors_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/contributors","subscribers_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/subscribers","subscription_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/subscription","commits_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/contents/{+path}","compare_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/merges","archive_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/downloads","issues_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/issues{/number}","pulls_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/pulls{/number}","milestones_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/milestones{/number}","notifications_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/labels{/name}","releases_url":"https://api.github.com/repos/notthatbreezy/vagrant-geotrellis-mesos-spark/releases{/id}","created_at":"2014-12-30T16:11:11Z","updated_at":"2015-02-13T20:24:32Z","pushed_at":"2015-04-23T11:53:51Z","git_url":"git://github.com/notthatbreezy/vagrant-geotrellis-mesos-spark.git","ssh_url":"git@github.com:notthatbreezy/vagrant-geotrellis-mesos-spark.git","clone_url":"https://github.com/notthatbreezy/vagrant-geotrellis-mesos-spark.git","svn_url":"https://github.com/notthatbreezy/vagrant-geotrellis-mesos-spark","homepage":null,"size":226,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"develop"}},"base":{"label":"geotrellis:develop","ref":"develop","sha":"8aa38d1d9b12e1a9b371eeb693a7721c7666ae3d","user":{"login":"geotrellis","id":3129444,"avatar_url":"https://avatars.githubusercontent.com/u/3129444?v=3","gravatar_id":"","url":"https://api.github.com/users/geotrellis","html_url":"https://github.com/geotrellis","followers_url":"https://api.github.com/users/geotrellis/followers","following_url":"https://api.github.com/users/geotrellis/following{/other_user}","gists_url":"https://api.github.com/users/geotrellis/gists{/gist_id}","starred_url":"https://api.github.com/users/geotrellis/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/geotrellis/subscriptions","organizations_url":"https://api.github.com/users/geotrellis/orgs","repos_url":"https://api.github.com/users/geotrellis/repos","events_url":"https://api.github.com/users/geotrellis/events{/privacy}","received_events_url":"https://api.github.com/users/geotrellis/received_events","type":"Organization","site_admin":false},"repo":{"id":27147158,"name":"geotrellis-ec2-cluster","full_name":"geotrellis/geotrellis-ec2-cluster","owner":{"login":"geotrellis","id":3129444,"avatar_url":"https://avatars.githubusercontent.com/u/3129444?v=3","gravatar_id":"","url":"https://api.github.com/users/geotrellis","html_url":"https://github.com/geotrellis","followers_url":"https://api.github.com/users/geotrellis/followers","following_url":"https://api.github.com/users/geotrellis/following{/other_user}","gists_url":"https://api.github.com/users/geotrellis/gists{/gist_id}","starred_url":"https://api.github.com/users/geotrellis/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/geotrellis/subscriptions","organizations_url":"https://api.github.com/users/geotrellis/orgs","repos_url":"https://api.github.com/users/geotrellis/repos","events_url":"https://api.github.com/users/geotrellis/events{/privacy}","received_events_url":"https://api.github.com/users/geotrellis/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/geotrellis/geotrellis-ec2-cluster","description":"Scripts to deploy a GeoTrellis Spark cluster on EC2","fork":false,"url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster","forks_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/forks","keys_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/keys{/key_id}","collaborators_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/teams","hooks_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/hooks","issue_events_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/issues/events{/number}","events_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/events","assignees_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/assignees{/user}","branches_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/branches{/branch}","tags_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/tags","blobs_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/git/refs{/sha}","trees_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/git/trees{/sha}","statuses_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/statuses/{sha}","languages_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/languages","stargazers_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/stargazers","contributors_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/contributors","subscribers_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/subscribers","subscription_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/subscription","commits_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/commits{/sha}","git_commits_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/git/commits{/sha}","comments_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/comments{/number}","issue_comment_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/issues/comments{/number}","contents_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/contents/{+path}","compare_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/compare/{base}...{head}","merges_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/merges","archive_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/downloads","issues_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/issues{/number}","pulls_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/pulls{/number}","milestones_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/milestones{/number}","notifications_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/labels{/name}","releases_url":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/releases{/id}","created_at":"2014-11-25T21:38:01Z","updated_at":"2015-04-20T19:14:47Z","pushed_at":"2015-04-23T11:59:23Z","git_url":"git://github.com/geotrellis/geotrellis-ec2-cluster.git","ssh_url":"git@github.com:geotrellis/geotrellis-ec2-cluster.git","clone_url":"https://github.com/geotrellis/geotrellis-ec2-cluster.git","svn_url":"https://github.com/geotrellis/geotrellis-ec2-cluster","homepage":"","size":649,"stargazers_count":2,"watchers_count":2,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":3,"mirror_url":null,"open_issues_count":8,"forks":3,"open_issues":8,"watchers":2,"default_branch":"develop"}},"_links":{"self":{"href":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/pulls/26"},"html":{"href":"https://github.com/geotrellis/geotrellis-ec2-cluster/pull/26"},"issue":{"href":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/issues/26"},"comments":{"href":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/issues/26/comments"},"review_comments":{"href":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/pulls/26/comments"},"review_comment":{"href":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/pulls/26/commits"},"statuses":{"href":"https://api.github.com/repos/geotrellis/geotrellis-ec2-cluster/statuses/aacd69970f42118268f3e3052aa6e8e3e99b520f"}}}}