{"action":"created","comment":{"url":"https://api.github.com/repos/Kitware/romanesco/pulls/comments/32023559","id":32023559,"diff_hunk":"@@ -0,0 +1,48 @@\n+import six\n+import romanesco\n+import os\n+import sys\n+\n+from ConfigParser import ConfigParser, NoOptionError\n+\n+def setup_spark_env():\n+    # Setup pyspark\n+    try:\n+        spark_home = romanesco.config.get('spark', 'spark_home')\n+\n+        # If not configured try the environment\n+        if not spark_home:\n+            spark_home = os.environ.get('SPARK_HOME')\n+\n+        if not spark_home:\n+            raise Exception('spark_home must be set or SPARK_HOME must be set in the environment')\n+\n+        # Need to set SPARK_HOME\n+        os.environ['SPARK_HOME'] = spark_home\n+\n+        if not os.path.exists(spark_home):\n+            raise Exception('spark_home is not a valid directory')\n+    except NoOptionError:\n+        raise Exception('spark_home must be configured')\n+\n+    sys.path.append(os.path.join(spark_home, 'python'))\n+    sys.path.append(os.path.join(spark_home, 'bin'))\n+\n+    # Check that we can import SparkContext\n+    try:\n+        from pyspark import SparkConf, SparkContext\n+    except Exception as ex:\n+        six.raise_from(Exception('Unable to create SparkContext, check Spark installation'), ex)\n+\n+\n+def create_spark_context():\n+    from pyspark import SparkConf, SparkContext\n+    # Set can spark configuration parameter user has specified\n+    spark_config = SparkConf();\n+    for (name, value) in romanesco.config.items('spark'):\n+        spark_config.set(name, value)","path":"romanesco/spark.py","position":43,"original_position":43,"commit_id":"027c085613021e14a308352490f2391726bcede9","original_commit_id":"027c085613021e14a308352490f2391726bcede9","user":{"login":"cjh1","id":1031053,"avatar_url":"https://avatars.githubusercontent.com/u/1031053?v=3","gravatar_id":"","url":"https://api.github.com/users/cjh1","html_url":"https://github.com/cjh1","followers_url":"https://api.github.com/users/cjh1/followers","following_url":"https://api.github.com/users/cjh1/following{/other_user}","gists_url":"https://api.github.com/users/cjh1/gists{/gist_id}","starred_url":"https://api.github.com/users/cjh1/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cjh1/subscriptions","organizations_url":"https://api.github.com/users/cjh1/orgs","repos_url":"https://api.github.com/users/cjh1/repos","events_url":"https://api.github.com/users/cjh1/events{/privacy}","received_events_url":"https://api.github.com/users/cjh1/received_events","type":"User","site_admin":false},"body":"Not at all, my view is review early and often :-) I appreciate the feedback.","created_at":"2015-06-09T15:11:20Z","updated_at":"2015-06-09T15:11:20Z","html_url":"https://github.com/Kitware/romanesco/pull/82#discussion_r32023559","pull_request_url":"https://api.github.com/repos/Kitware/romanesco/pulls/82","_links":{"self":{"href":"https://api.github.com/repos/Kitware/romanesco/pulls/comments/32023559"},"html":{"href":"https://github.com/Kitware/romanesco/pull/82#discussion_r32023559"},"pull_request":{"href":"https://api.github.com/repos/Kitware/romanesco/pulls/82"}}},"pull_request":{"url":"https://api.github.com/repos/Kitware/romanesco/pulls/82","id":37258884,"html_url":"https://github.com/Kitware/romanesco/pull/82","diff_url":"https://github.com/Kitware/romanesco/pull/82.diff","patch_url":"https://github.com/Kitware/romanesco/pull/82.patch","issue_url":"https://api.github.com/repos/Kitware/romanesco/issues/82","number":82,"state":"open","locked":false,"title":"[WIP] Add suppose for running Spark jobs in romanesco","user":{"login":"cjh1","id":1031053,"avatar_url":"https://avatars.githubusercontent.com/u/1031053?v=3","gravatar_id":"","url":"https://api.github.com/users/cjh1","html_url":"https://github.com/cjh1","followers_url":"https://api.github.com/users/cjh1/followers","following_url":"https://api.github.com/users/cjh1/following{/other_user}","gists_url":"https://api.github.com/users/cjh1/gists{/gist_id}","starred_url":"https://api.github.com/users/cjh1/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cjh1/subscriptions","organizations_url":"https://api.github.com/users/cjh1/orgs","repos_url":"https://api.github.com/users/cjh1/repos","events_url":"https://api.github.com/users/cjh1/events{/privacy}","received_events_url":"https://api.github.com/users/cjh1/received_events","type":"User","site_admin":false},"body":"","created_at":"2015-06-09T14:26:29Z","updated_at":"2015-06-09T15:11:20Z","closed_at":null,"merged_at":null,"merge_commit_sha":"a0ddd73876a795d4f41469e3e285dcac50a66a0c","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/Kitware/romanesco/pulls/82/commits","review_comments_url":"https://api.github.com/repos/Kitware/romanesco/pulls/82/comments","review_comment_url":"https://api.github.com/repos/Kitware/romanesco/pulls/comments{/number}","comments_url":"https://api.github.com/repos/Kitware/romanesco/issues/82/comments","statuses_url":"https://api.github.com/repos/Kitware/romanesco/statuses/027c085613021e14a308352490f2391726bcede9","head":{"label":"cjh1:spark","ref":"spark","sha":"027c085613021e14a308352490f2391726bcede9","user":{"login":"cjh1","id":1031053,"avatar_url":"https://avatars.githubusercontent.com/u/1031053?v=3","gravatar_id":"","url":"https://api.github.com/users/cjh1","html_url":"https://github.com/cjh1","followers_url":"https://api.github.com/users/cjh1/followers","following_url":"https://api.github.com/users/cjh1/following{/other_user}","gists_url":"https://api.github.com/users/cjh1/gists{/gist_id}","starred_url":"https://api.github.com/users/cjh1/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cjh1/subscriptions","organizations_url":"https://api.github.com/users/cjh1/orgs","repos_url":"https://api.github.com/users/cjh1/repos","events_url":"https://api.github.com/users/cjh1/events{/privacy}","received_events_url":"https://api.github.com/users/cjh1/received_events","type":"User","site_admin":false},"repo":{"id":37136727,"name":"romanesco","full_name":"cjh1/romanesco","owner":{"login":"cjh1","id":1031053,"avatar_url":"https://avatars.githubusercontent.com/u/1031053?v=3","gravatar_id":"","url":"https://api.github.com/users/cjh1","html_url":"https://github.com/cjh1","followers_url":"https://api.github.com/users/cjh1/followers","following_url":"https://api.github.com/users/cjh1/following{/other_user}","gists_url":"https://api.github.com/users/cjh1/gists{/gist_id}","starred_url":"https://api.github.com/users/cjh1/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cjh1/subscriptions","organizations_url":"https://api.github.com/users/cjh1/orgs","repos_url":"https://api.github.com/users/cjh1/repos","events_url":"https://api.github.com/users/cjh1/events{/privacy}","received_events_url":"https://api.github.com/users/cjh1/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/cjh1/romanesco","description":"A simple analytics execution engine.","fork":true,"url":"https://api.github.com/repos/cjh1/romanesco","forks_url":"https://api.github.com/repos/cjh1/romanesco/forks","keys_url":"https://api.github.com/repos/cjh1/romanesco/keys{/key_id}","collaborators_url":"https://api.github.com/repos/cjh1/romanesco/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/cjh1/romanesco/teams","hooks_url":"https://api.github.com/repos/cjh1/romanesco/hooks","issue_events_url":"https://api.github.com/repos/cjh1/romanesco/issues/events{/number}","events_url":"https://api.github.com/repos/cjh1/romanesco/events","assignees_url":"https://api.github.com/repos/cjh1/romanesco/assignees{/user}","branches_url":"https://api.github.com/repos/cjh1/romanesco/branches{/branch}","tags_url":"https://api.github.com/repos/cjh1/romanesco/tags","blobs_url":"https://api.github.com/repos/cjh1/romanesco/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/cjh1/romanesco/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/cjh1/romanesco/git/refs{/sha}","trees_url":"https://api.github.com/repos/cjh1/romanesco/git/trees{/sha}","statuses_url":"https://api.github.com/repos/cjh1/romanesco/statuses/{sha}","languages_url":"https://api.github.com/repos/cjh1/romanesco/languages","stargazers_url":"https://api.github.com/repos/cjh1/romanesco/stargazers","contributors_url":"https://api.github.com/repos/cjh1/romanesco/contributors","subscribers_url":"https://api.github.com/repos/cjh1/romanesco/subscribers","subscription_url":"https://api.github.com/repos/cjh1/romanesco/subscription","commits_url":"https://api.github.com/repos/cjh1/romanesco/commits{/sha}","git_commits_url":"https://api.github.com/repos/cjh1/romanesco/git/commits{/sha}","comments_url":"https://api.github.com/repos/cjh1/romanesco/comments{/number}","issue_comment_url":"https://api.github.com/repos/cjh1/romanesco/issues/comments{/number}","contents_url":"https://api.github.com/repos/cjh1/romanesco/contents/{+path}","compare_url":"https://api.github.com/repos/cjh1/romanesco/compare/{base}...{head}","merges_url":"https://api.github.com/repos/cjh1/romanesco/merges","archive_url":"https://api.github.com/repos/cjh1/romanesco/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/cjh1/romanesco/downloads","issues_url":"https://api.github.com/repos/cjh1/romanesco/issues{/number}","pulls_url":"https://api.github.com/repos/cjh1/romanesco/pulls{/number}","milestones_url":"https://api.github.com/repos/cjh1/romanesco/milestones{/number}","notifications_url":"https://api.github.com/repos/cjh1/romanesco/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/cjh1/romanesco/labels{/name}","releases_url":"https://api.github.com/repos/cjh1/romanesco/releases{/id}","created_at":"2015-06-09T14:24:09Z","updated_at":"2015-06-09T14:24:10Z","pushed_at":"2015-06-09T14:47:07Z","git_url":"git://github.com/cjh1/romanesco.git","ssh_url":"git@github.com:cjh1/romanesco.git","clone_url":"https://github.com/cjh1/romanesco.git","svn_url":"https://github.com/cjh1/romanesco","homepage":"http://www.tangelohub.org/romanesco/","size":2427,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"Kitware:gaia-compat","ref":"gaia-compat","sha":"bfda9b1251ad718232d5f7c08fbaf666d54dad0c","user":{"login":"Kitware","id":87549,"avatar_url":"https://avatars.githubusercontent.com/u/87549?v=3","gravatar_id":"","url":"https://api.github.com/users/Kitware","html_url":"https://github.com/Kitware","followers_url":"https://api.github.com/users/Kitware/followers","following_url":"https://api.github.com/users/Kitware/following{/other_user}","gists_url":"https://api.github.com/users/Kitware/gists{/gist_id}","starred_url":"https://api.github.com/users/Kitware/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Kitware/subscriptions","organizations_url":"https://api.github.com/users/Kitware/orgs","repos_url":"https://api.github.com/users/Kitware/repos","events_url":"https://api.github.com/users/Kitware/events{/privacy}","received_events_url":"https://api.github.com/users/Kitware/received_events","type":"Organization","site_admin":false},"repo":{"id":18351718,"name":"romanesco","full_name":"Kitware/romanesco","owner":{"login":"Kitware","id":87549,"avatar_url":"https://avatars.githubusercontent.com/u/87549?v=3","gravatar_id":"","url":"https://api.github.com/users/Kitware","html_url":"https://github.com/Kitware","followers_url":"https://api.github.com/users/Kitware/followers","following_url":"https://api.github.com/users/Kitware/following{/other_user}","gists_url":"https://api.github.com/users/Kitware/gists{/gist_id}","starred_url":"https://api.github.com/users/Kitware/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Kitware/subscriptions","organizations_url":"https://api.github.com/users/Kitware/orgs","repos_url":"https://api.github.com/users/Kitware/repos","events_url":"https://api.github.com/users/Kitware/events{/privacy}","received_events_url":"https://api.github.com/users/Kitware/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/Kitware/romanesco","description":"A simple analytics execution engine.","fork":false,"url":"https://api.github.com/repos/Kitware/romanesco","forks_url":"https://api.github.com/repos/Kitware/romanesco/forks","keys_url":"https://api.github.com/repos/Kitware/romanesco/keys{/key_id}","collaborators_url":"https://api.github.com/repos/Kitware/romanesco/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/Kitware/romanesco/teams","hooks_url":"https://api.github.com/repos/Kitware/romanesco/hooks","issue_events_url":"https://api.github.com/repos/Kitware/romanesco/issues/events{/number}","events_url":"https://api.github.com/repos/Kitware/romanesco/events","assignees_url":"https://api.github.com/repos/Kitware/romanesco/assignees{/user}","branches_url":"https://api.github.com/repos/Kitware/romanesco/branches{/branch}","tags_url":"https://api.github.com/repos/Kitware/romanesco/tags","blobs_url":"https://api.github.com/repos/Kitware/romanesco/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/Kitware/romanesco/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/Kitware/romanesco/git/refs{/sha}","trees_url":"https://api.github.com/repos/Kitware/romanesco/git/trees{/sha}","statuses_url":"https://api.github.com/repos/Kitware/romanesco/statuses/{sha}","languages_url":"https://api.github.com/repos/Kitware/romanesco/languages","stargazers_url":"https://api.github.com/repos/Kitware/romanesco/stargazers","contributors_url":"https://api.github.com/repos/Kitware/romanesco/contributors","subscribers_url":"https://api.github.com/repos/Kitware/romanesco/subscribers","subscription_url":"https://api.github.com/repos/Kitware/romanesco/subscription","commits_url":"https://api.github.com/repos/Kitware/romanesco/commits{/sha}","git_commits_url":"https://api.github.com/repos/Kitware/romanesco/git/commits{/sha}","comments_url":"https://api.github.com/repos/Kitware/romanesco/comments{/number}","issue_comment_url":"https://api.github.com/repos/Kitware/romanesco/issues/comments{/number}","contents_url":"https://api.github.com/repos/Kitware/romanesco/contents/{+path}","compare_url":"https://api.github.com/repos/Kitware/romanesco/compare/{base}...{head}","merges_url":"https://api.github.com/repos/Kitware/romanesco/merges","archive_url":"https://api.github.com/repos/Kitware/romanesco/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/Kitware/romanesco/downloads","issues_url":"https://api.github.com/repos/Kitware/romanesco/issues{/number}","pulls_url":"https://api.github.com/repos/Kitware/romanesco/pulls{/number}","milestones_url":"https://api.github.com/repos/Kitware/romanesco/milestones{/number}","notifications_url":"https://api.github.com/repos/Kitware/romanesco/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/Kitware/romanesco/labels{/name}","releases_url":"https://api.github.com/repos/Kitware/romanesco/releases{/id}","created_at":"2014-04-02T02:10:24Z","updated_at":"2015-06-01T18:08:13Z","pushed_at":"2015-06-09T14:47:07Z","git_url":"git://github.com/Kitware/romanesco.git","ssh_url":"git@github.com:Kitware/romanesco.git","clone_url":"https://github.com/Kitware/romanesco.git","svn_url":"https://github.com/Kitware/romanesco","homepage":"http://www.tangelohub.org/romanesco/","size":2427,"stargazers_count":6,"watchers_count":6,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":4,"mirror_url":null,"open_issues_count":23,"forks":4,"open_issues":23,"watchers":6,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/Kitware/romanesco/pulls/82"},"html":{"href":"https://github.com/Kitware/romanesco/pull/82"},"issue":{"href":"https://api.github.com/repos/Kitware/romanesco/issues/82"},"comments":{"href":"https://api.github.com/repos/Kitware/romanesco/issues/82/comments"},"review_comments":{"href":"https://api.github.com/repos/Kitware/romanesco/pulls/82/comments"},"review_comment":{"href":"https://api.github.com/repos/Kitware/romanesco/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/Kitware/romanesco/pulls/82/commits"},"statuses":{"href":"https://api.github.com/repos/Kitware/romanesco/statuses/027c085613021e14a308352490f2391726bcede9"}}}}