{"action":"created","comment":{"url":"https://api.github.com/repos/Kitware/romanesco/pulls/comments/32248997","id":32248997,"diff_hunk":"@@ -0,0 +1,66 @@\n+import romanesco\n+import unittest\n+import os\n+\n+\n+class TestSpark(unittest.TestCase):\n+\n+    def setUp(self):\n+        self.prevdir = os.getcwd()\n+        cur_path = os.path.dirname(os.path.realpath(__file__))\n+        os.chdir(cur_path)\n+\n+    def testJsonToSparkRdd(self):\n+        analysis = {\n+            'name': 'reduce',\n+            'inputs': [{'name': 'a', 'type': 'collection', 'format': 'spark.rdd'}],\n+            'outputs': [{'name': 'b', 'type': 'number', 'format': 'number'}],\n+            'mode': 'spark.python',\n+            'spark_conf': {\n+                'spark.app.name': 'test_add',\n+                'spark.master': os.environ['SPARK_TEST_MASTER_URL']\n+            }\n+        }\n+\n+        with open('data/spark_add_script.py', 'r') as fp:\n+            script = fp.read()\n+            analysis['script'] = script\n+\n+        outputs = romanesco.run(analysis, {\n+                'a': {'format': 'json', 'data': '[1,2,3,4,5,6,7,8,9]'}\n+            },\n+            {\n+                'b': {'format': 'number'}\n+            })\n+\n+        expected = {'b': {'data': 45, 'format': 'number'}}\n+        self.assertEqual(outputs, expected)\n+\n+    def testSparkRddToJson(self):\n+        analysis = {\n+            'name': 'map',\n+            'inputs': [{'name': 'a', 'type': 'collection', 'format': 'spark.rdd'}],\n+            'outputs': [{'name': 'b', 'type': 'collection', 'format': 'spark.rdd'}],\n+            'mode': 'spark.python',\n+            'spark_conf': {\n+                'spark.app.name': 'test_square',\n+                'spark.master': os.environ['SPARK_TEST_MASTER_URL']\n+            }\n+        }\n+\n+        with open('data/spark_square_script.py', 'r') as fp:\n+            script = fp.read()\n+            analysis['script'] = script\n+\n+        outputs = romanesco.run(analysis, {\n+                'a': {'format': 'json', 'data': '[1,2,3,4,5,6,7,8,9]'}\n+            },\n+            {\n+                'b': {'format': 'json'}\n+            })\n+\n+        expected = {'b': {'data': '[1, 4, 9, 16, 25, 36, 49, 64, 81]', 'format': 'json'}}","path":"tests/spark_test.py","position":62,"original_position":62,"commit_id":"228432d0304cfa0787256b71b45fd2e1489ffe55","original_commit_id":"228432d0304cfa0787256b71b45fd2e1489ffe55","user":{"login":"jbeezley","id":31890,"avatar_url":"https://avatars.githubusercontent.com/u/31890?v=3","gravatar_id":"","url":"https://api.github.com/users/jbeezley","html_url":"https://github.com/jbeezley","followers_url":"https://api.github.com/users/jbeezley/followers","following_url":"https://api.github.com/users/jbeezley/following{/other_user}","gists_url":"https://api.github.com/users/jbeezley/gists{/gist_id}","starred_url":"https://api.github.com/users/jbeezley/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jbeezley/subscriptions","organizations_url":"https://api.github.com/users/jbeezley/orgs","repos_url":"https://api.github.com/users/jbeezley/repos","events_url":"https://api.github.com/users/jbeezley/events{/privacy}","received_events_url":"https://api.github.com/users/jbeezley/received_events","type":"User","site_admin":false},"body":"It gets converted automagically because of line 59 above.","created_at":"2015-06-11T17:57:18Z","updated_at":"2015-06-11T17:57:18Z","html_url":"https://github.com/Kitware/romanesco/pull/83#discussion_r32248997","pull_request_url":"https://api.github.com/repos/Kitware/romanesco/pulls/83","_links":{"self":{"href":"https://api.github.com/repos/Kitware/romanesco/pulls/comments/32248997"},"html":{"href":"https://github.com/Kitware/romanesco/pull/83#discussion_r32248997"},"pull_request":{"href":"https://api.github.com/repos/Kitware/romanesco/pulls/83"}}},"pull_request":{"url":"https://api.github.com/repos/Kitware/romanesco/pulls/83","id":37274280,"html_url":"https://github.com/Kitware/romanesco/pull/83","diff_url":"https://github.com/Kitware/romanesco/pull/83.diff","patch_url":"https://github.com/Kitware/romanesco/pull/83.patch","issue_url":"https://api.github.com/repos/Kitware/romanesco/issues/83","number":83,"state":"open","locked":false,"title":"Add support for running Spark jobs in romanesco","user":{"login":"cjh1","id":1031053,"avatar_url":"https://avatars.githubusercontent.com/u/1031053?v=3","gravatar_id":"","url":"https://api.github.com/users/cjh1","html_url":"https://github.com/cjh1","followers_url":"https://api.github.com/users/cjh1/followers","following_url":"https://api.github.com/users/cjh1/following{/other_user}","gists_url":"https://api.github.com/users/cjh1/gists{/gist_id}","starred_url":"https://api.github.com/users/cjh1/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cjh1/subscriptions","organizations_url":"https://api.github.com/users/cjh1/orgs","repos_url":"https://api.github.com/users/cjh1/repos","events_url":"https://api.github.com/users/cjh1/events{/privacy}","received_events_url":"https://api.github.com/users/cjh1/received_events","type":"User","site_admin":false},"body":"","created_at":"2015-06-09T16:45:30Z","updated_at":"2015-06-11T17:57:18Z","closed_at":null,"merged_at":null,"merge_commit_sha":"240c98aa11a291a77027892f5fc4a980e2f7041b","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/Kitware/romanesco/pulls/83/commits","review_comments_url":"https://api.github.com/repos/Kitware/romanesco/pulls/83/comments","review_comment_url":"https://api.github.com/repos/Kitware/romanesco/pulls/comments{/number}","comments_url":"https://api.github.com/repos/Kitware/romanesco/issues/83/comments","statuses_url":"https://api.github.com/repos/Kitware/romanesco/statuses/228432d0304cfa0787256b71b45fd2e1489ffe55","head":{"label":"cjh1:spark","ref":"spark","sha":"228432d0304cfa0787256b71b45fd2e1489ffe55","user":{"login":"cjh1","id":1031053,"avatar_url":"https://avatars.githubusercontent.com/u/1031053?v=3","gravatar_id":"","url":"https://api.github.com/users/cjh1","html_url":"https://github.com/cjh1","followers_url":"https://api.github.com/users/cjh1/followers","following_url":"https://api.github.com/users/cjh1/following{/other_user}","gists_url":"https://api.github.com/users/cjh1/gists{/gist_id}","starred_url":"https://api.github.com/users/cjh1/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cjh1/subscriptions","organizations_url":"https://api.github.com/users/cjh1/orgs","repos_url":"https://api.github.com/users/cjh1/repos","events_url":"https://api.github.com/users/cjh1/events{/privacy}","received_events_url":"https://api.github.com/users/cjh1/received_events","type":"User","site_admin":false},"repo":{"id":37136727,"name":"romanesco","full_name":"cjh1/romanesco","owner":{"login":"cjh1","id":1031053,"avatar_url":"https://avatars.githubusercontent.com/u/1031053?v=3","gravatar_id":"","url":"https://api.github.com/users/cjh1","html_url":"https://github.com/cjh1","followers_url":"https://api.github.com/users/cjh1/followers","following_url":"https://api.github.com/users/cjh1/following{/other_user}","gists_url":"https://api.github.com/users/cjh1/gists{/gist_id}","starred_url":"https://api.github.com/users/cjh1/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cjh1/subscriptions","organizations_url":"https://api.github.com/users/cjh1/orgs","repos_url":"https://api.github.com/users/cjh1/repos","events_url":"https://api.github.com/users/cjh1/events{/privacy}","received_events_url":"https://api.github.com/users/cjh1/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/cjh1/romanesco","description":"A simple analytics execution engine.","fork":true,"url":"https://api.github.com/repos/cjh1/romanesco","forks_url":"https://api.github.com/repos/cjh1/romanesco/forks","keys_url":"https://api.github.com/repos/cjh1/romanesco/keys{/key_id}","collaborators_url":"https://api.github.com/repos/cjh1/romanesco/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/cjh1/romanesco/teams","hooks_url":"https://api.github.com/repos/cjh1/romanesco/hooks","issue_events_url":"https://api.github.com/repos/cjh1/romanesco/issues/events{/number}","events_url":"https://api.github.com/repos/cjh1/romanesco/events","assignees_url":"https://api.github.com/repos/cjh1/romanesco/assignees{/user}","branches_url":"https://api.github.com/repos/cjh1/romanesco/branches{/branch}","tags_url":"https://api.github.com/repos/cjh1/romanesco/tags","blobs_url":"https://api.github.com/repos/cjh1/romanesco/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/cjh1/romanesco/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/cjh1/romanesco/git/refs{/sha}","trees_url":"https://api.github.com/repos/cjh1/romanesco/git/trees{/sha}","statuses_url":"https://api.github.com/repos/cjh1/romanesco/statuses/{sha}","languages_url":"https://api.github.com/repos/cjh1/romanesco/languages","stargazers_url":"https://api.github.com/repos/cjh1/romanesco/stargazers","contributors_url":"https://api.github.com/repos/cjh1/romanesco/contributors","subscribers_url":"https://api.github.com/repos/cjh1/romanesco/subscribers","subscription_url":"https://api.github.com/repos/cjh1/romanesco/subscription","commits_url":"https://api.github.com/repos/cjh1/romanesco/commits{/sha}","git_commits_url":"https://api.github.com/repos/cjh1/romanesco/git/commits{/sha}","comments_url":"https://api.github.com/repos/cjh1/romanesco/comments{/number}","issue_comment_url":"https://api.github.com/repos/cjh1/romanesco/issues/comments{/number}","contents_url":"https://api.github.com/repos/cjh1/romanesco/contents/{+path}","compare_url":"https://api.github.com/repos/cjh1/romanesco/compare/{base}...{head}","merges_url":"https://api.github.com/repos/cjh1/romanesco/merges","archive_url":"https://api.github.com/repos/cjh1/romanesco/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/cjh1/romanesco/downloads","issues_url":"https://api.github.com/repos/cjh1/romanesco/issues{/number}","pulls_url":"https://api.github.com/repos/cjh1/romanesco/pulls{/number}","milestones_url":"https://api.github.com/repos/cjh1/romanesco/milestones{/number}","notifications_url":"https://api.github.com/repos/cjh1/romanesco/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/cjh1/romanesco/labels{/name}","releases_url":"https://api.github.com/repos/cjh1/romanesco/releases{/id}","created_at":"2015-06-09T14:24:09Z","updated_at":"2015-06-09T14:24:10Z","pushed_at":"2015-06-10T19:01:48Z","git_url":"git://github.com/cjh1/romanesco.git","ssh_url":"git@github.com:cjh1/romanesco.git","clone_url":"https://github.com/cjh1/romanesco.git","svn_url":"https://github.com/cjh1/romanesco","homepage":"http://www.tangelohub.org/romanesco/","size":2427,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"Kitware:master","ref":"master","sha":"c55eb8869914dc522284de311c989677f6ede140","user":{"login":"Kitware","id":87549,"avatar_url":"https://avatars.githubusercontent.com/u/87549?v=3","gravatar_id":"","url":"https://api.github.com/users/Kitware","html_url":"https://github.com/Kitware","followers_url":"https://api.github.com/users/Kitware/followers","following_url":"https://api.github.com/users/Kitware/following{/other_user}","gists_url":"https://api.github.com/users/Kitware/gists{/gist_id}","starred_url":"https://api.github.com/users/Kitware/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Kitware/subscriptions","organizations_url":"https://api.github.com/users/Kitware/orgs","repos_url":"https://api.github.com/users/Kitware/repos","events_url":"https://api.github.com/users/Kitware/events{/privacy}","received_events_url":"https://api.github.com/users/Kitware/received_events","type":"Organization","site_admin":false},"repo":{"id":18351718,"name":"romanesco","full_name":"Kitware/romanesco","owner":{"login":"Kitware","id":87549,"avatar_url":"https://avatars.githubusercontent.com/u/87549?v=3","gravatar_id":"","url":"https://api.github.com/users/Kitware","html_url":"https://github.com/Kitware","followers_url":"https://api.github.com/users/Kitware/followers","following_url":"https://api.github.com/users/Kitware/following{/other_user}","gists_url":"https://api.github.com/users/Kitware/gists{/gist_id}","starred_url":"https://api.github.com/users/Kitware/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Kitware/subscriptions","organizations_url":"https://api.github.com/users/Kitware/orgs","repos_url":"https://api.github.com/users/Kitware/repos","events_url":"https://api.github.com/users/Kitware/events{/privacy}","received_events_url":"https://api.github.com/users/Kitware/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/Kitware/romanesco","description":"A simple analytics execution engine.","fork":false,"url":"https://api.github.com/repos/Kitware/romanesco","forks_url":"https://api.github.com/repos/Kitware/romanesco/forks","keys_url":"https://api.github.com/repos/Kitware/romanesco/keys{/key_id}","collaborators_url":"https://api.github.com/repos/Kitware/romanesco/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/Kitware/romanesco/teams","hooks_url":"https://api.github.com/repos/Kitware/romanesco/hooks","issue_events_url":"https://api.github.com/repos/Kitware/romanesco/issues/events{/number}","events_url":"https://api.github.com/repos/Kitware/romanesco/events","assignees_url":"https://api.github.com/repos/Kitware/romanesco/assignees{/user}","branches_url":"https://api.github.com/repos/Kitware/romanesco/branches{/branch}","tags_url":"https://api.github.com/repos/Kitware/romanesco/tags","blobs_url":"https://api.github.com/repos/Kitware/romanesco/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/Kitware/romanesco/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/Kitware/romanesco/git/refs{/sha}","trees_url":"https://api.github.com/repos/Kitware/romanesco/git/trees{/sha}","statuses_url":"https://api.github.com/repos/Kitware/romanesco/statuses/{sha}","languages_url":"https://api.github.com/repos/Kitware/romanesco/languages","stargazers_url":"https://api.github.com/repos/Kitware/romanesco/stargazers","contributors_url":"https://api.github.com/repos/Kitware/romanesco/contributors","subscribers_url":"https://api.github.com/repos/Kitware/romanesco/subscribers","subscription_url":"https://api.github.com/repos/Kitware/romanesco/subscription","commits_url":"https://api.github.com/repos/Kitware/romanesco/commits{/sha}","git_commits_url":"https://api.github.com/repos/Kitware/romanesco/git/commits{/sha}","comments_url":"https://api.github.com/repos/Kitware/romanesco/comments{/number}","issue_comment_url":"https://api.github.com/repos/Kitware/romanesco/issues/comments{/number}","contents_url":"https://api.github.com/repos/Kitware/romanesco/contents/{+path}","compare_url":"https://api.github.com/repos/Kitware/romanesco/compare/{base}...{head}","merges_url":"https://api.github.com/repos/Kitware/romanesco/merges","archive_url":"https://api.github.com/repos/Kitware/romanesco/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/Kitware/romanesco/downloads","issues_url":"https://api.github.com/repos/Kitware/romanesco/issues{/number}","pulls_url":"https://api.github.com/repos/Kitware/romanesco/pulls{/number}","milestones_url":"https://api.github.com/repos/Kitware/romanesco/milestones{/number}","notifications_url":"https://api.github.com/repos/Kitware/romanesco/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/Kitware/romanesco/labels{/name}","releases_url":"https://api.github.com/repos/Kitware/romanesco/releases{/id}","created_at":"2014-04-02T02:10:24Z","updated_at":"2015-06-01T18:08:13Z","pushed_at":"2015-06-10T19:01:49Z","git_url":"git://github.com/Kitware/romanesco.git","ssh_url":"git@github.com:Kitware/romanesco.git","clone_url":"https://github.com/Kitware/romanesco.git","svn_url":"https://github.com/Kitware/romanesco","homepage":"http://www.tangelohub.org/romanesco/","size":2427,"stargazers_count":6,"watchers_count":6,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":4,"mirror_url":null,"open_issues_count":25,"forks":4,"open_issues":25,"watchers":6,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/Kitware/romanesco/pulls/83"},"html":{"href":"https://github.com/Kitware/romanesco/pull/83"},"issue":{"href":"https://api.github.com/repos/Kitware/romanesco/issues/83"},"comments":{"href":"https://api.github.com/repos/Kitware/romanesco/issues/83/comments"},"review_comments":{"href":"https://api.github.com/repos/Kitware/romanesco/pulls/83/comments"},"review_comment":{"href":"https://api.github.com/repos/Kitware/romanesco/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/Kitware/romanesco/pulls/83/commits"},"statuses":{"href":"https://api.github.com/repos/Kitware/romanesco/statuses/228432d0304cfa0787256b71b45fd2e1489ffe55"}}}}