{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/32875303","id":32875303,"diff_hunk":"@@ -28,6 +29,241 @@\n USER_HOME = os.environ.get(\"HOME\")\n \n \n+# -------------------------------------------------------------------------------------------------\n+# Test module definitions and functions for traversing module dependency graph\n+# -------------------------------------------------------------------------------------------------\n+\n+\n+all_modules = []\n+\n+\n+class Module(object):\n+\n+    def __init__(self, name, dependencies, source_file_regexes, sbt_test_goals=(),\n+                 should_run_python_tests=False, should_run_r_tests=False):\n+        self.name = name\n+        self.dependencies = dependencies\n+        self.source_file_prefixes = source_file_regexes\n+        self.sbt_test_goals = sbt_test_goals\n+        self.should_run_python_tests = should_run_python_tests\n+        self.should_run_r_tests = should_run_r_tests\n+\n+        self.dependent_modules = set()\n+        for dep in dependencies:\n+            dep.dependent_modules.add(self)\n+        all_modules.append(self)\n+\n+    def contains_file(self, filename):\n+        return any(re.match(p, filename) for p in self.source_file_prefixes)\n+\n+\n+root = Module(\n+    name=\"root\",\n+    dependencies=[],\n+    source_file_regexes=[],\n+    sbt_test_goals=[\n+        \"test\",\n+    ],\n+    should_run_python_tests=True,\n+    should_run_r_tests=True\n+)\n+\n+\n+sql = Module(\n+    name=\"sql\",\n+    dependencies=[],\n+    source_file_regexes=[\n+        \"sql/(?!hive-thriftserver)\",\n+        \"bin/spark-sql\",\n+        \"examples/src/main/java/org/apache/spark/examples/sql/\",\n+        \"examples/src/main/scala/org/apache/spark/examples/sql/\",\n+    ],\n+    sbt_test_goals=[\n+        \"catalyst/test\",\n+        \"sql/test\",\n+        \"hive/test\",\n+    ])\n+\n+\n+hive_thriftserver = Module(\n+    name=\"hive-thriftserver\",\n+    dependencies=[sql],\n+    source_file_regexes=[\n+        \"sql/hive-thriftserver\",\n+        \"sbin/start-thriftserver.sh\",\n+    ],\n+    sbt_test_goals=[\n+        \"hive-thriftserver/test\",\n+    ]\n+)\n+\n+\n+graphx = Module(\n+    name=\"graphx\",\n+    dependencies=[],\n+    source_file_regexes=[\n+        \"graphx/\",\n+    ],\n+    sbt_test_goals=[\n+        \"graphx/test\"\n+    ]\n+)\n+\n+\n+streaming = Module(\n+    name=\"streaming\",\n+    dependencies=[],\n+    source_file_regexes=[\n+        \"external/\",\n+        \"extras/java8-tests/\",\n+        \"extras/kinesis-asl/\",\n+        \"streaming\",","path":"dev/run-tests.py","position":100,"original_position":100,"commit_id":"e46539fb9140d2b0c1d6d616a6b9d4c4f697d061","original_commit_id":"e46539fb9140d2b0c1d6d616a6b9d4c4f697d061","user":{"login":"JoshRosen","id":50748,"avatar_url":"https://avatars.githubusercontent.com/u/50748?v=3","gravatar_id":"","url":"https://api.github.com/users/JoshRosen","html_url":"https://github.com/JoshRosen","followers_url":"https://api.github.com/users/JoshRosen/followers","following_url":"https://api.github.com/users/JoshRosen/following{/other_user}","gists_url":"https://api.github.com/users/JoshRosen/gists{/gist_id}","starred_url":"https://api.github.com/users/JoshRosen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/JoshRosen/subscriptions","organizations_url":"https://api.github.com/users/JoshRosen/orgs","repos_url":"https://api.github.com/users/JoshRosen/repos","events_url":"https://api.github.com/users/JoshRosen/events{/privacy}","received_events_url":"https://api.github.com/users/JoshRosen/received_events","type":"User","site_admin":false},"body":"these should be covered by the examples subproject instead.","created_at":"2015-06-19T22:37:33Z","updated_at":"2015-06-19T22:37:33Z","html_url":"https://github.com/apache/spark/pull/6866#discussion_r32875303","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/6866","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/32875303"},"html":{"href":"https://github.com/apache/spark/pull/6866#discussion_r32875303"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/6866"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/6866","id":37963579,"html_url":"https://github.com/apache/spark/pull/6866","diff_url":"https://github.com/apache/spark/pull/6866.diff","patch_url":"https://github.com/apache/spark/pull/6866.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/6866","number":6866,"state":"open","locked":false,"title":"[SPARK-8422] [BUILD] [PROJECT INFRA] Add a module abstraction to dev/run-tests","user":{"login":"JoshRosen","id":50748,"avatar_url":"https://avatars.githubusercontent.com/u/50748?v=3","gravatar_id":"","url":"https://api.github.com/users/JoshRosen","html_url":"https://github.com/JoshRosen","followers_url":"https://api.github.com/users/JoshRosen/followers","following_url":"https://api.github.com/users/JoshRosen/following{/other_user}","gists_url":"https://api.github.com/users/JoshRosen/gists{/gist_id}","starred_url":"https://api.github.com/users/JoshRosen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/JoshRosen/subscriptions","organizations_url":"https://api.github.com/users/JoshRosen/orgs","repos_url":"https://api.github.com/users/JoshRosen/repos","events_url":"https://api.github.com/users/JoshRosen/events{/privacy}","received_events_url":"https://api.github.com/users/JoshRosen/received_events","type":"User","site_admin":false},"body":"This patch builds upon #5694 to add a 'module' abstraction to the `dev/run-tests` script which groups together the per-module test logic, including the mapping from file paths to modules, the mapping from modules to test goals, and the dependencies / relationships between modules.\r\n\r\nThis refactoring makes it much easier to increase the granularity of test modules, which will let us skip even more tests.  It's also a prerequisite for other changes that will reduce test time, such as running subsets of the Python tests based on which files / modules have changed.\r\n\r\nThis patch also adds doctests for the new graph traversal / change mapping code.","created_at":"2015-06-18T01:17:47Z","updated_at":"2015-06-19T22:37:33Z","closed_at":null,"merged_at":null,"merge_commit_sha":"5cf6ab9d2a77a234fa86c32eb816fa574104eb12","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/6866/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/6866/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/spark/issues/6866/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/e46539fb9140d2b0c1d6d616a6b9d4c4f697d061","head":{"label":"JoshRosen:more-dev-run-tests-refactoring","ref":"more-dev-run-tests-refactoring","sha":"e46539fb9140d2b0c1d6d616a6b9d4c4f697d061","user":{"login":"JoshRosen","id":50748,"avatar_url":"https://avatars.githubusercontent.com/u/50748?v=3","gravatar_id":"","url":"https://api.github.com/users/JoshRosen","html_url":"https://github.com/JoshRosen","followers_url":"https://api.github.com/users/JoshRosen/followers","following_url":"https://api.github.com/users/JoshRosen/following{/other_user}","gists_url":"https://api.github.com/users/JoshRosen/gists{/gist_id}","starred_url":"https://api.github.com/users/JoshRosen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/JoshRosen/subscriptions","organizations_url":"https://api.github.com/users/JoshRosen/orgs","repos_url":"https://api.github.com/users/JoshRosen/repos","events_url":"https://api.github.com/users/JoshRosen/events{/privacy}","received_events_url":"https://api.github.com/users/JoshRosen/received_events","type":"User","site_admin":false},"repo":{"id":22238710,"name":"spark","full_name":"JoshRosen/spark","owner":{"login":"JoshRosen","id":50748,"avatar_url":"https://avatars.githubusercontent.com/u/50748?v=3","gravatar_id":"","url":"https://api.github.com/users/JoshRosen","html_url":"https://github.com/JoshRosen","followers_url":"https://api.github.com/users/JoshRosen/followers","following_url":"https://api.github.com/users/JoshRosen/following{/other_user}","gists_url":"https://api.github.com/users/JoshRosen/gists{/gist_id}","starred_url":"https://api.github.com/users/JoshRosen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/JoshRosen/subscriptions","organizations_url":"https://api.github.com/users/JoshRosen/orgs","repos_url":"https://api.github.com/users/JoshRosen/repos","events_url":"https://api.github.com/users/JoshRosen/events{/privacy}","received_events_url":"https://api.github.com/users/JoshRosen/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/JoshRosen/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/JoshRosen/spark","forks_url":"https://api.github.com/repos/JoshRosen/spark/forks","keys_url":"https://api.github.com/repos/JoshRosen/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/JoshRosen/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/JoshRosen/spark/teams","hooks_url":"https://api.github.com/repos/JoshRosen/spark/hooks","issue_events_url":"https://api.github.com/repos/JoshRosen/spark/issues/events{/number}","events_url":"https://api.github.com/repos/JoshRosen/spark/events","assignees_url":"https://api.github.com/repos/JoshRosen/spark/assignees{/user}","branches_url":"https://api.github.com/repos/JoshRosen/spark/branches{/branch}","tags_url":"https://api.github.com/repos/JoshRosen/spark/tags","blobs_url":"https://api.github.com/repos/JoshRosen/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/JoshRosen/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/JoshRosen/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/JoshRosen/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/JoshRosen/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/JoshRosen/spark/languages","stargazers_url":"https://api.github.com/repos/JoshRosen/spark/stargazers","contributors_url":"https://api.github.com/repos/JoshRosen/spark/contributors","subscribers_url":"https://api.github.com/repos/JoshRosen/spark/subscribers","subscription_url":"https://api.github.com/repos/JoshRosen/spark/subscription","commits_url":"https://api.github.com/repos/JoshRosen/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/JoshRosen/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/JoshRosen/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/JoshRosen/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/JoshRosen/spark/contents/{+path}","compare_url":"https://api.github.com/repos/JoshRosen/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/JoshRosen/spark/merges","archive_url":"https://api.github.com/repos/JoshRosen/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/JoshRosen/spark/downloads","issues_url":"https://api.github.com/repos/JoshRosen/spark/issues{/number}","pulls_url":"https://api.github.com/repos/JoshRosen/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/JoshRosen/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/JoshRosen/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/JoshRosen/spark/labels{/name}","releases_url":"https://api.github.com/repos/JoshRosen/spark/releases{/id}","created_at":"2014-07-25T00:47:52Z","updated_at":"2015-05-01T22:45:32Z","pushed_at":"2015-06-19T06:50:02Z","git_url":"git://github.com/JoshRosen/spark.git","ssh_url":"git@github.com:JoshRosen/spark.git","clone_url":"https://github.com/JoshRosen/spark.git","svn_url":"https://github.com/JoshRosen/spark","homepage":null,"size":111572,"stargazers_count":1,"watchers_count":1,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":3,"mirror_url":null,"open_issues_count":0,"forks":3,"open_issues":0,"watchers":1,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"e86fbdb1e6f1538f65ef78d90bbc41604f6bd580","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"name":"spark","full_name":"apache/spark","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","created_at":"2014-02-25T08:00:08Z","updated_at":"2015-06-19T21:12:56Z","pushed_at":"2015-06-19T22:13:31Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":1470905,"stargazers_count":4452,"watchers_count":4452,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":3915,"mirror_url":"git://git.apache.org/spark.git","open_issues_count":352,"forks":3915,"open_issues":352,"watchers":4452,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/6866"},"html":{"href":"https://github.com/apache/spark/pull/6866"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/6866"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/6866/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/6866/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/6866/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/e46539fb9140d2b0c1d6d616a6b9d4c4f697d061"}}}}